diff options
739 files changed, 21136 insertions, 7870 deletions
diff --git a/.clippy.toml b/.clippy.toml new file mode 100644 index 000000000000..e4c4eef10b28 --- /dev/null +++ b/.clippy.toml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 + +check-private-items = true + +disallowed-macros = [ + # The `clippy::dbg_macro` lint only works with `std::dbg!`, thus we simulate + # it here, see: https://github.com/rust-lang/rust-clippy/issues/11303. + { path = "kernel::dbg", reason = "the `dbg!` macro is intended as a debugging tool" }, +] diff --git a/.gitignore b/.gitignore index 56972adb5031..a61e4778d011 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ modules.order # We don't want to ignore the following even if they are dot-files # !.clang-format +!.clippy.toml !.cocciconfig !.editorconfig !.get_maintainer.ignore diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci index 7f63c7e97773..5da6a14dc326 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci +++ b/Documentation/ABI/testing/sysfs-bus-pci @@ -163,6 +163,17 @@ Description: will be present in sysfs. Writing 1 to this file will perform reset. +What: /sys/bus/pci/devices/.../reset_subordinate +Date: October 2024 +Contact: linux-pci@vger.kernel.org +Description: + This is visible only for bridge devices. If you want to reset + all devices attached through the subordinate bus of a specific + bridge device, writing 1 to this will try to do it. This will + affect all devices attached to the system through this bridge + similiar to writing 1 to their individual "reset" file, so use + with caution. + What: /sys/bus/pci/devices/.../vpd Date: February 2008 Contact: Ben Hutchings <bwh@kernel.org> diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index fdedf1ea944b..3e1630c70d8a 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -311,10 +311,13 @@ Description: Do background GC aggressively when set. Set to 0 by default. GC approach and turns SSR mode on. gc urgent low(2): lowers the bar of checking I/O idling in order to process outstanding discard commands and GC a - little bit aggressively. uses cost benefit GC approach. + little bit aggressively. always uses cost benefit GC approach, + and will override age-threshold GC approach if ATGC is enabled + at the same time. gc urgent mid(3): does GC forcibly in a period of given gc_urgent_sleep_time and executes a mid level of I/O idling check. - uses cost benefit GC approach. + always uses cost benefit GC approach, and will override + age-threshold GC approach if ATGC is enabled at the same time. What: /sys/fs/f2fs/<disk>/gc_urgent_sleep_time Date: August 2017 @@ -819,3 +822,9 @@ Description: It controls the valid block ratio threshold not to trigger excessiv for zoned deivces. The initial value of it is 95(%). F2FS will stop the background GC thread from intiating GC for sections having valid blocks exceeding the ratio. + +What: /sys/fs/f2fs/<disk>/max_read_extent_count +Date: November 2024 +Contact: "Chao Yu" <chao@kernel.org> +Description: It controls max read extent count for per-inode, the value of threshold + is 10240 by default. diff --git a/Documentation/PCI/endpoint/pci-endpoint.rst b/Documentation/PCI/endpoint/pci-endpoint.rst index 21507e3cc238..35f82f2d45f5 100644 --- a/Documentation/PCI/endpoint/pci-endpoint.rst +++ b/Documentation/PCI/endpoint/pci-endpoint.rst @@ -117,6 +117,35 @@ by the PCI endpoint function driver. The PCI endpoint function driver should use pci_epc_mem_free_addr() to free the memory space allocated using pci_epc_mem_alloc_addr(). +* pci_epc_map_addr() + + A PCI endpoint function driver should use pci_epc_map_addr() to map to a RC + PCI address the CPU address of local memory obtained with + pci_epc_mem_alloc_addr(). + +* pci_epc_unmap_addr() + + A PCI endpoint function driver should use pci_epc_unmap_addr() to unmap the + CPU address of local memory mapped to a RC address with pci_epc_map_addr(). + +* pci_epc_mem_map() + + A PCI endpoint controller may impose constraints on the RC PCI addresses that + can be mapped. The function pci_epc_mem_map() allows endpoint function + drivers to allocate and map controller memory while handling such + constraints. This function will determine the size of the memory that must be + allocated with pci_epc_mem_alloc_addr() for successfully mapping a RC PCI + address range. This function will also indicate the size of the PCI address + range that was actually mapped, which can be less than the requested size, as + well as the offset into the allocated memory to use for accessing the mapped + RC PCI address range. + +* pci_epc_mem_unmap() + + A PCI endpoint function driver can use pci_epc_mem_unmap() to unmap and free + controller memory that was allocated and mapped using pci_epc_mem_map(). + + Other EPC APIs ~~~~~~~~~~~~~~ diff --git a/Documentation/PCI/index.rst b/Documentation/PCI/index.rst index e73f84aebde3..5e7c4e6e726b 100644 --- a/Documentation/PCI/index.rst +++ b/Documentation/PCI/index.rst @@ -18,3 +18,4 @@ PCI Bus Subsystem pcieaer-howto endpoint/index boot-interrupts + tph diff --git a/Documentation/PCI/pciebus-howto.rst b/Documentation/PCI/pciebus-howto.rst index f344452651e1..375d9ce171f6 100644 --- a/Documentation/PCI/pciebus-howto.rst +++ b/Documentation/PCI/pciebus-howto.rst @@ -217,8 +217,12 @@ capability structure except the PCI Express capability structure, that is shared between many drivers including the service drivers. RMW Capability accessors (pcie_capability_clear_and_set_word(), pcie_capability_set_word(), and pcie_capability_clear_word()) protect -a selected set of PCI Express Capability Registers (Link Control -Register and Root Control Register). Any change to those registers -should be performed using RMW accessors to avoid problems due to -concurrent updates. For the up-to-date list of protected registers, -see pcie_capability_clear_and_set_word(). +a selected set of PCI Express Capability Registers: + +* Link Control Register +* Root Control Register +* Link Control 2 Register + +Any change to those registers should be performed using RMW accessors to +avoid problems due to concurrent updates. For the up-to-date list of +protected registers, see pcie_capability_clear_and_set_word(). diff --git a/Documentation/PCI/tph.rst b/Documentation/PCI/tph.rst new file mode 100644 index 000000000000..e8993be64fd6 --- /dev/null +++ b/Documentation/PCI/tph.rst @@ -0,0 +1,132 @@ +.. SPDX-License-Identifier: GPL-2.0 + + +=========== +TPH Support +=========== + +:Copyright: 2024 Advanced Micro Devices, Inc. +:Authors: - Eric van Tassell <eric.vantassell@amd.com> + - Wei Huang <wei.huang2@amd.com> + + +Overview +======== + +TPH (TLP Processing Hints) is a PCIe feature that allows endpoint devices +to provide optimization hints for requests that target memory space. +These hints, in a format called Steering Tags (STs), are embedded in the +requester's TLP headers, enabling the system hardware, such as the Root +Complex, to better manage platform resources for these requests. + +For example, on platforms with TPH-based direct data cache injection +support, an endpoint device can include appropriate STs in its DMA +traffic to specify which cache the data should be written to. This allows +the CPU core to have a higher probability of getting data from cache, +potentially improving performance and reducing latency in data +processing. + + +How to Use TPH +============== + +TPH is presented as an optional extended capability in PCIe. The Linux +kernel handles TPH discovery during boot, but it is up to the device +driver to request TPH enablement if it is to be utilized. Once enabled, +the driver uses the provided API to obtain the Steering Tag for the +target memory and to program the ST into the device's ST table. + +Enable TPH support in Linux +--------------------------- + +To support TPH, the kernel must be built with the CONFIG_PCIE_TPH option +enabled. + +Manage TPH +---------- + +To enable TPH for a device, use the following function:: + + int pcie_enable_tph(struct pci_dev *pdev, int mode); + +This function enables TPH support for device with a specific ST mode. +Current supported modes include: + + * PCI_TPH_ST_NS_MODE - NO ST Mode + * PCI_TPH_ST_IV_MODE - Interrupt Vector Mode + * PCI_TPH_ST_DS_MODE - Device Specific Mode + +`pcie_enable_tph()` checks whether the requested mode is actually +supported by the device before enabling. The device driver can figure out +which TPH mode is supported and can be properly enabled based on the +return value of `pcie_enable_tph()`. + +To disable TPH, use the following function:: + + void pcie_disable_tph(struct pci_dev *pdev); + +Manage ST +--------- + +Steering Tags are platform specific. PCIe spec does not specify where STs +are from. Instead PCI Firmware Specification defines an ACPI _DSM method +(see the `Revised _DSM for Cache Locality TPH Features ECN +<https://members.pcisig.com/wg/PCI-SIG/document/15470>`_) for retrieving +STs for a target memory of various properties. This method is what is +supported in this implementation. + +To retrieve a Steering Tag for a target memory associated with a specific +CPU, use the following function:: + + int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type type, + unsigned int cpu_uid, u16 *tag); + +The `type` argument is used to specify the memory type, either volatile +or persistent, of the target memory. The `cpu_uid` argument specifies the +CPU where the memory is associated to. + +After the ST value is retrieved, the device driver can use the following +function to write the ST into the device:: + + int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, + u16 tag); + +The `index` argument is the ST table entry index the ST tag will be +written into. `pcie_tph_set_st_entry()` will figure out the proper +location of ST table, either in the MSI-X table or in the TPH Extended +Capability space, and write the Steering Tag into the ST entry pointed by +the `index` argument. + +It is completely up to the driver to decide how to use these TPH +functions. For example a network device driver can use the TPH APIs above +to update the Steering Tag when interrupt affinity of a RX/TX queue has +been changed. Here is a sample code for IRQ affinity notifier: + +.. code-block:: c + + static void irq_affinity_notified(struct irq_affinity_notify *notify, + const cpumask_t *mask) + { + struct drv_irq *irq; + unsigned int cpu_id; + u16 tag; + + irq = container_of(notify, struct drv_irq, affinity_notify); + cpumask_copy(irq->cpu_mask, mask); + + /* Pick a right CPU as the target - here is just an example */ + cpu_id = cpumask_first(irq->cpu_mask); + + if (pcie_tph_get_cpu_st(irq->pdev, TPH_MEM_TYPE_VM, cpu_id, + &tag)) + return; + + if (pcie_tph_set_st_entry(irq->pdev, irq->msix_nr, tag)) + return; + } + +Disable TPH system-wide +----------------------- + +There is a kernel command line option available to control TPH feature: + * "notph": TPH will be disabled for all endpoint devices. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 062ea4adb51d..14ebd1f4b45e 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4686,6 +4686,10 @@ nomio [S390] Do not use MIO instructions. norid [S390] ignore the RID field and force use of one PCI domain per PCI function + notph [PCIE] If the PCIE_TPH kernel config parameter + is enabled, this kernel boot option can be used + to disable PCIe TLP Processing Hints support + system-wide. pcie_aspm= [PCIE] Forcibly enable or ignore PCIe Active State Power Management. diff --git a/Documentation/admin-guide/media/index.rst b/Documentation/admin-guide/media/index.rst index be7e0e4482ca..b11737ae6c04 100644 --- a/Documentation/admin-guide/media/index.rst +++ b/Documentation/admin-guide/media/index.rst @@ -20,6 +20,11 @@ Documentation/driver-api/media/index.rst - for driver development information and Kernel APIs used by media devices; +Documentation/process/debugging/media_specific_debugging_guide.rst + + - for advice about essential tools and techniques to debug drivers on this + subsystem + .. toctree:: :caption: Table of Contents :maxdepth: 2 diff --git a/Documentation/admin-guide/sysctl/fs.rst b/Documentation/admin-guide/sysctl/fs.rst index 30c61474dec5..f5ec6c9312e1 100644 --- a/Documentation/admin-guide/sysctl/fs.rst +++ b/Documentation/admin-guide/sysctl/fs.rst @@ -337,3 +337,13 @@ Each "watch" costs roughly 90 bytes on a 32-bit kernel, and roughly 160 bytes on a 64-bit one. The current default value for ``max_user_watches`` is 4% of the available low memory, divided by the "watch" cost in bytes. + +5. /proc/sys/fs/fuse - Configuration options for FUSE filesystems +===================================================================== + +This directory contains the following configuration options for FUSE +filesystems: + +``/proc/sys/fs/fuse/max_pages_limit`` is a read/write file for +setting/getting the maximum number of pages that can be used for servicing +requests in FUSE. diff --git a/Documentation/core-api/gfp_mask-from-fs-io.rst b/Documentation/core-api/gfp_mask-from-fs-io.rst index e7c32a8de126..858b2fbcb36c 100644 --- a/Documentation/core-api/gfp_mask-from-fs-io.rst +++ b/Documentation/core-api/gfp_mask-from-fs-io.rst @@ -55,14 +55,16 @@ scope. What about __vmalloc(GFP_NOFS) ============================== -vmalloc doesn't support GFP_NOFS semantic because there are hardcoded -GFP_KERNEL allocations deep inside the allocator which are quite non-trivial -to fix up. That means that calling ``vmalloc`` with GFP_NOFS/GFP_NOIO is -almost always a bug. The good news is that the NOFS/NOIO semantic can be -achieved by the scope API. +Since v5.17, and specifically after the commit 451769ebb7e79 ("mm/vmalloc: +alloc GFP_NO{FS,IO} for vmalloc"), GFP_NOFS/GFP_NOIO are now supported in +``[k]vmalloc`` by implicitly using scope API. + +In earlier kernels ``vmalloc`` didn't support GFP_NOFS semantic because there +were hardcoded GFP_KERNEL allocations deep inside the allocator. That means +that calling ``vmalloc`` with GFP_NOFS/GFP_NOIO was almost always a bug. In the ideal world, upper layers should already mark dangerous contexts -and so no special care is required and vmalloc should be called without -any problems. Sometimes if the context is not really clear or there are -layering violations then the recommended way around that is to wrap ``vmalloc`` -by the scope API with a comment explaining the problem. +and so no special care is required and ``vmalloc`` should be called without any +problems. Sometimes if the context is not really clear or there are layering +violations then the recommended way around that (on pre-v5.17 kernels) is to +wrap ``vmalloc`` by the scope API with a comment explaining the problem. diff --git a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml index 898c1be2d6a4..f05aab2b1add 100644 --- a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml +++ b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml @@ -149,7 +149,7 @@ allOf: then: properties: clocks: - minItems: 4 + minItems: 6 clock-names: items: @@ -178,7 +178,7 @@ allOf: then: properties: clocks: - minItems: 4 + minItems: 6 clock-names: items: @@ -207,6 +207,7 @@ allOf: properties: clocks: minItems: 4 + maxItems: 4 clock-names: items: diff --git a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml index 612633ba59e2..2e1547569702 100644 --- a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml +++ b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml @@ -17,6 +17,12 @@ properties: compatible: const: microchip,pcie-host-1.0 # PolarFire + reg: + minItems: 3 + + reg-names: + minItems: 3 + clocks: description: Fabric Interface Controllers, FICs, are the interface between the FPGA @@ -62,8 +68,9 @@ examples: pcie0: pcie@2030000000 { compatible = "microchip,pcie-host-1.0"; reg = <0x0 0x70000000 0x0 0x08000000>, - <0x0 0x43000000 0x0 0x00010000>; - reg-names = "cfg", "apb"; + <0x0 0x43008000 0x0 0x00002000>, + <0x0 0x4300a000 0x0 0x00002000>; + reg-names = "cfg", "bridge", "ctrl"; device_type = "pci"; #address-cells = <3>; #size-cells = <2>; diff --git a/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi-common.yaml b/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi-common.yaml index 7a57a80052a0..039eecdbd6aa 100644 --- a/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi-common.yaml +++ b/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi-common.yaml @@ -18,12 +18,18 @@ allOf: properties: reg: - maxItems: 2 + maxItems: 3 + minItems: 2 reg-names: - items: - - const: cfg - - const: apb + oneOf: + - items: + - const: cfg + - const: apb + - items: + - const: cfg + - const: bridge + - const: ctrl interrupts: minItems: 1 diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml index e18900c41576..0480c58f7d99 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-common.yaml @@ -81,6 +81,10 @@ properties: vddpe-3v3-supply: description: PCIe endpoint power supply + operating-points-v2: true + opp-table: + type: object + required: - reg - reg-names diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml index 46bd59eefadb..6e0a6d8f0ed0 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8450.yaml @@ -70,10 +70,6 @@ properties: - const: msi7 - const: global - operating-points-v2: true - opp-table: - type: object - resets: maxItems: 1 diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml index 24cb38673581..2b5498a35dcc 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml @@ -20,6 +20,7 @@ properties: - const: qcom,pcie-sm8550 - items: - enum: + - qcom,sar2130p-pcie - qcom,pcie-sm8650 - const: qcom,pcie-sm8550 @@ -39,7 +40,7 @@ properties: clocks: minItems: 7 - maxItems: 8 + maxItems: 9 clock-names: minItems: 7 @@ -52,6 +53,7 @@ properties: - const: ddrss_sf_tbu # PCIe SF TBU clock - const: noc_aggr # Aggre NoC PCIe AXI clock - const: cnoc_sf_axi # Config NoC PCIe1 AXI clock + - const: qmip_pcie_ahb # QMIP PCIe AHB clock interrupts: minItems: 8 diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml index a9db0a231563..257068a18264 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-x1e80100.yaml @@ -47,9 +47,10 @@ properties: interrupts: minItems: 8 - maxItems: 8 + maxItems: 9 interrupt-names: + minItems: 8 items: - const: msi0 - const: msi1 @@ -59,6 +60,7 @@ properties: - const: msi5 - const: msi6 - const: msi7 + - const: global resets: minItems: 1 @@ -130,9 +132,10 @@ examples: <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>; + <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "msi0", "msi1", "msi2", "msi3", - "msi4", "msi5", "msi6", "msi7"; + "msi4", "msi5", "msi6", "msi7", "global"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml index ffabbac57fc1..bd87f6b49d68 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml @@ -26,6 +26,7 @@ properties: - qcom,pcie-ipq8064-v2 - qcom,pcie-ipq8074 - qcom,pcie-ipq8074-gen3 + - qcom,pcie-ipq9574 - qcom,pcie-msm8996 - qcom,pcie-qcs404 - qcom,pcie-sdm845 @@ -164,6 +165,7 @@ allOf: enum: - qcom,pcie-ipq6018 - qcom,pcie-ipq8074-gen3 + - qcom,pcie-ipq9574 then: properties: reg: @@ -405,6 +407,53 @@ allOf: compatible: contains: enum: + - qcom,pcie-ipq9574 + then: + properties: + clocks: + minItems: 6 + maxItems: 6 + clock-names: + items: + - const: axi_m # AXI Master clock + - const: axi_s # AXI Slave clock + - const: axi_bridge + - const: rchng + - const: ahb + - const: aux + + resets: + minItems: 8 + maxItems: 8 + reset-names: + items: + - const: pipe # PIPE reset + - const: sticky # Core Sticky reset + - const: axi_s_sticky # AXI Slave Sticky reset + - const: axi_s # AXI Slave reset + - const: axi_m_sticky # AXI Master Sticky reset + - const: axi_m # AXI Master reset + - const: aux # AUX Reset + - const: ahb # AHB Reset + + interrupts: + minItems: 8 + interrupt-names: + items: + - const: msi0 + - const: msi1 + - const: msi2 + - const: msi3 + - const: msi4 + - const: msi5 + - const: msi6 + - const: msi7 + + - if: + properties: + compatible: + contains: + enum: - qcom,pcie-qcs404 then: properties: @@ -510,6 +559,7 @@ allOf: - qcom,pcie-ipq8064v2 - qcom,pcie-ipq8074 - qcom,pcie-ipq8074-gen3 + - qcom,pcie-ipq9574 - qcom,pcie-qcs404 then: required: diff --git a/Documentation/devicetree/bindings/pci/snps,dw-pcie.yaml b/Documentation/devicetree/bindings/pci/snps,dw-pcie.yaml index 548f59d76ef2..205326fb2d75 100644 --- a/Documentation/devicetree/bindings/pci/snps,dw-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/snps,dw-pcie.yaml @@ -230,7 +230,6 @@ examples: interrupts = <25>, <24>; interrupt-names = "msi", "hp"; - #interrupt-cells = <1>; reset-gpios = <&port0 0 1>; diff --git a/Documentation/devicetree/bindings/pci/starfive,jh7110-pcie.yaml b/Documentation/devicetree/bindings/pci/starfive,jh7110-pcie.yaml index 67151aaa3948..5f432452c815 100644 --- a/Documentation/devicetree/bindings/pci/starfive,jh7110-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/starfive,jh7110-pcie.yaml @@ -16,6 +16,13 @@ properties: compatible: const: starfive,jh7110-pcie + + reg: + maxItems: 2 + + reg-names: + maxItems: 2 + clocks: items: - description: NOC bus clock diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml index 4b9fb74fb9e9..fd3423e6051b 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml @@ -16,6 +16,7 @@ description: properties: compatible: enum: + - qcom,sar2130p-adsp-pas - qcom,sm8350-adsp-pas - qcom,sm8350-cdsp-pas - qcom,sm8350-slpi-pas @@ -61,6 +62,7 @@ allOf: properties: compatible: enum: + - qcom,sar2130p-adsp-pas - qcom,sm8350-adsp-pas - qcom,sm8350-cdsp-pas - qcom,sm8350-slpi-pas @@ -101,6 +103,7 @@ allOf: properties: compatible: enum: + - qcom,sar2130p-adsp-pas - qcom,sm8350-adsp-pas - qcom,sm8350-slpi-pas - qcom,sm8450-adsp-pas diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml index d7fad7b3c2c6..a24cbb61bda7 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml @@ -15,16 +15,20 @@ description: properties: compatible: - enum: - - qcom,sdx75-mpss-pas - - qcom,sm8550-adsp-pas - - qcom,sm8550-cdsp-pas - - qcom,sm8550-mpss-pas - - qcom,sm8650-adsp-pas - - qcom,sm8650-cdsp-pas - - qcom,sm8650-mpss-pas - - qcom,x1e80100-adsp-pas - - qcom,x1e80100-cdsp-pas + oneOf: + - enum: + - qcom,sdx75-mpss-pas + - qcom,sm8550-adsp-pas + - qcom,sm8550-cdsp-pas + - qcom,sm8550-mpss-pas + - qcom,sm8650-adsp-pas + - qcom,sm8650-cdsp-pas + - qcom,sm8650-mpss-pas + - qcom,x1e80100-adsp-pas + - qcom,x1e80100-cdsp-pas + - items: + - const: qcom,sm8750-adsp-pas + - const: qcom,sm8550-adsp-pas reg: maxItems: 1 @@ -85,6 +89,20 @@ allOf: - if: properties: compatible: + contains: + enum: + - qcom,sm8750-adsp-pas + then: + properties: + interrupts: + maxItems: 6 + interrupt-names: + maxItems: 6 + memory-region: + maxItems: 2 + - if: + properties: + compatible: enum: - qcom,sm8650-cdsp-pas then: @@ -129,10 +147,12 @@ allOf: - if: properties: compatible: - enum: - - qcom,sm8550-adsp-pas - - qcom,sm8650-adsp-pas - - qcom,x1e80100-adsp-pas + contains: + enum: + - qcom,sm8550-adsp-pas + - qcom,sm8650-adsp-pas + - qcom,sm8750-adsp-pas + - qcom,x1e80100-adsp-pas then: properties: power-domains: diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst index e6ffd59bb8f0..af9697e60165 100644 --- a/Documentation/doc-guide/kernel-doc.rst +++ b/Documentation/doc-guide/kernel-doc.rst @@ -533,6 +533,7 @@ identifiers: *[ function/type ...]* Include documentation for each *function* and *type* in *source*. If no *function* is specified, the documentation for all functions and types in the *source* will be included. + *type* can be a struct, union, enum, or typedef identifier. Examples:: diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index ebbf8e4cc85f..568ed9714dbe 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -394,7 +394,6 @@ PCI pcim_enable_device() : after success, some PCI ops become managed pcim_iomap() : do iomap() on a single BAR pcim_iomap_regions() : do request_region() and iomap() on multiple BARs - pcim_iomap_regions_request_all() : do request_region() on all and iomap() on multiple BARs pcim_iomap_table() : array of mapped addresses indexed by BAR pcim_iounmap() : do iounmap() on a single BAR pcim_iounmap_regions() : do iounmap() and release_region() on multiple BARs diff --git a/Documentation/driver-api/pci/pci.rst b/Documentation/driver-api/pci/pci.rst index aa40b1cc243b..59d86e827198 100644 --- a/Documentation/driver-api/pci/pci.rst +++ b/Documentation/driver-api/pci/pci.rst @@ -46,6 +46,9 @@ PCI Support Library .. kernel-doc:: drivers/pci/pci-sysfs.c :internal: +.. kernel-doc:: drivers/pci/tph.c + :export: + PCI Hotplug Support Library --------------------------- diff --git a/Documentation/driver-api/pwrseq.rst b/Documentation/driver-api/pwrseq.rst index a644084ded17..ad18b2326b68 100644 --- a/Documentation/driver-api/pwrseq.rst +++ b/Documentation/driver-api/pwrseq.rst @@ -11,7 +11,7 @@ Introduction ============ This framework is designed to abstract complex power-up sequences that are -shared between multiple logical devices in the linux kernel. +shared between multiple logical devices in the Linux kernel. The intention is to allow consumers to obtain a power sequencing handle exposed by the power sequence provider and delegate the actual requesting and @@ -25,7 +25,7 @@ The power sequencing API uses a number of terms specific to the subsystem: Unit - A unit is a discreet chunk of a power sequence. For instance one unit may + A unit is a discrete chunk of a power sequence. For instance one unit may enable a set of regulators, another may enable a specific GPIO. Units can define dependencies in the form of other units that must be enabled before it itself can be. @@ -62,7 +62,7 @@ Provider interface The provider API is admittedly not nearly as straightforward as the one for consumers but it makes up for it in flexibility. -Each provider can logically split the power-up sequence into descrete chunks +Each provider can logically split the power-up sequence into discrete chunks (units) and define their dependencies. They can then expose named targets that consumers may use as the final point in the sequence that they wish to reach. @@ -72,7 +72,7 @@ register with the pwrseq subsystem by calling pwrseq_device_register(). Dynamic consumer matching ------------------------- -The main difference between pwrseq and other linux kernel providers is the +The main difference between pwrseq and other Linux kernel providers is the mechanism for dynamic matching of consumers and providers. Every power sequence provider driver must implement the `match()` callback and pass it to the pwrseq core when registering with the subsystems. diff --git a/Documentation/filesystems/autofs.rst b/Documentation/filesystems/autofs.rst index 1ac576458c69..5eb02394fcc3 100644 --- a/Documentation/filesystems/autofs.rst +++ b/Documentation/filesystems/autofs.rst @@ -442,7 +442,7 @@ which can be used to communicate directly with the autofs filesystem. It requires CAP_SYS_ADMIN for access. The 'ioctl's that can be used on this device are described in a separate -document `autofs-mount-control.txt`, and are summarised briefly here. +document `autofs-mount-control.rst`, and are summarised briefly here. Each ioctl is passed a pointer to an `autofs_dev_ioctl` structure:: struct autofs_dev_ioctl { diff --git a/Documentation/filesystems/dlmfs.rst b/Documentation/filesystems/dlmfs.rst index 7e2b1fd471d7..70d4e48242c3 100644 --- a/Documentation/filesystems/dlmfs.rst +++ b/Documentation/filesystems/dlmfs.rst @@ -36,7 +36,7 @@ None Usage ===== -If you're just interested in OCFS2, then please see ocfs2.txt. The +If you're just interested in OCFS2, then please see ocfs2.rst. The rest of this document will be geared towards those who want to use dlmfs for easy to setup and easy to use clustered locking in userspace. diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index 68a0885fb5e6..fb7d2ee022bc 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -943,3 +943,47 @@ NVMe Zoned Namespace devices can start before the zone-capacity and span across zone-capacity boundary. Such spanning segments are also considered as usable segments. All blocks past the zone-capacity are considered unusable in these segments. + +Device aliasing feature +----------------------- + +f2fs can utilize a special file called a "device aliasing file." This file allows +the entire storage device to be mapped with a single, large extent, not using +the usual f2fs node structures. This mapped area is pinned and primarily intended +for holding the space. + +Essentially, this mechanism allows a portion of the f2fs area to be temporarily +reserved and used by another filesystem or for different purposes. Once that +external usage is complete, the device aliasing file can be deleted, releasing +the reserved space back to F2FS for its own use. + +<use-case> + +# ls /dev/vd* +/dev/vdb (32GB) /dev/vdc (32GB) +# mkfs.ext4 /dev/vdc +# mkfs.f2fs -c /dev/vdc@vdc.file /dev/vdb +# mount /dev/vdb /mnt/f2fs +# ls -l /mnt/f2fs +vdc.file +# df -h +/dev/vdb 64G 33G 32G 52% /mnt/f2fs + +# mount -o loop /dev/vdc /mnt/ext4 +# df -h +/dev/vdb 64G 33G 32G 52% /mnt/f2fs +/dev/loop7 32G 24K 30G 1% /mnt/ext4 +# umount /mnt/ext4 + +# f2fs_io getflags /mnt/f2fs/vdc.file +get a flag on /mnt/f2fs/vdc.file ret=0, flags=nocow(pinned),immutable +# f2fs_io setflags noimmutable /mnt/f2fs/vdc.file +get a flag on noimmutable ret=0, flags=800010 +set a flag on /mnt/f2fs/vdc.file ret=0, flags=noimmutable +# rm /mnt/f2fs/vdc.file +# df -h +/dev/vdb 64G 753M 64G 2% /mnt/f2fs + +So, the key idea is, user can do any file operations on /dev/vdc, and +reclaim the space after the use, while the space is counted as /data. +That doesn't require modifying partition size and filesystem format. diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 0e2fac7a16da..76e538217868 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -16,7 +16,7 @@ btrfs filesystems. Like fscrypt, not too much filesystem-specific code is needed to support fs-verity. fs-verity is similar to `dm-verity -<https://www.kernel.org/doc/Documentation/device-mapper/verity.txt>`_ +<https://www.kernel.org/doc/Documentation/admin-guide/device-mapper/verity.rst>`_ but works on files rather than block devices. On regular files on filesystems supporting fs-verity, userspace can execute an ioctl that causes the filesystem to build a Merkle tree for the file and persist diff --git a/Documentation/filesystems/path-lookup.rst b/Documentation/filesystems/path-lookup.rst index 2b2df6aa5432..9ced1135608e 100644 --- a/Documentation/filesystems/path-lookup.rst +++ b/Documentation/filesystems/path-lookup.rst @@ -531,7 +531,7 @@ this retry process in the next article. Automount points are locations in the filesystem where an attempt to lookup a name can trigger changes to how that lookup should be handled, in particular by mounting a filesystem there. These are -covered in greater detail in autofs.txt in the Linux documentation +covered in greater detail in autofs.rst in the Linux documentation tree, but a few notes specifically related to path lookup are in order here. diff --git a/Documentation/filesystems/path-lookup.txt b/Documentation/filesystems/path-lookup.txt index 1aa7ce099f6f..d2cf2852e1f8 100644 --- a/Documentation/filesystems/path-lookup.txt +++ b/Documentation/filesystems/path-lookup.txt @@ -379,4 +379,4 @@ Papers and other documentation on dcache locking 2. http://lse.sourceforge.net/locking/dcache/dcache.html -3. path-lookup.md in this directory. +3. path-lookup.rst in this directory. diff --git a/Documentation/filesystems/ramfs-rootfs-initramfs.rst b/Documentation/filesystems/ramfs-rootfs-initramfs.rst index 447f767c6462..fa4f81099cb4 100644 --- a/Documentation/filesystems/ramfs-rootfs-initramfs.rst +++ b/Documentation/filesystems/ramfs-rootfs-initramfs.rst @@ -315,7 +315,7 @@ the above threads) is: 2) The cpio archive format chosen by the kernel is simpler and cleaner (and thus easier to create and parse) than any of the (literally dozens of) various tar archive formats. The complete initramfs archive format is - explained in buffer-format.txt, created in usr/gen_init_cpio.c, and + explained in buffer-format.rst, created in usr/gen_init_cpio.c, and extracted in init/initramfs.c. All three together come to less than 26k total of human-readable text. diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst index 1717348a4404..0042776a9e17 100644 --- a/Documentation/kernel-hacking/hacking.rst +++ b/Documentation/kernel-hacking/hacking.rst @@ -587,7 +587,7 @@ Defined in ``include/linux/export.h`` Similar to :c:func:`EXPORT_SYMBOL()` except that the symbols exported by :c:func:`EXPORT_SYMBOL_GPL()` can only be seen by -modules with a :c:func:`MODULE_LICENSE()` that specifies a GPL +modules with a :c:func:`MODULE_LICENSE()` that specifies a GPLv2 compatible license. It implies that the function is considered an internal implementation issue, and not really an interface. Some maintainers and developers may however require EXPORT_SYMBOL_GPL() diff --git a/Documentation/process/debugging/driver_development_debugging_guide.rst b/Documentation/process/debugging/driver_development_debugging_guide.rst new file mode 100644 index 000000000000..aef204094205 --- /dev/null +++ b/Documentation/process/debugging/driver_development_debugging_guide.rst @@ -0,0 +1,223 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================================== +Debugging advice for driver development +======================================== + +This document serves as a general starting point and lookup for debugging +device drivers. +While this guide focuses on debugging that requires re-compiling the +module/kernel, the :doc:`userspace debugging guide +</process/debugging/userspace_debugging_guide>` will guide +you through tools like dynamic debug, ftrace and other tools useful for +debugging issues and behavior. +For general debugging advice, see the :doc:`general advice document +</process/debugging/index>`. + +.. contents:: + :depth: 3 + +The following sections show you the available tools. + +printk() & friends +------------------ + +These are derivatives of printf() with varying destinations and support for +being dynamically turned on or off, or lack thereof. + +Simple printk() +~~~~~~~~~~~~~~~ + +The classic, can be used to great effect for quick and dirty development +of new modules or to extract arbitrary necessary data for troubleshooting. + +Prerequisite: ``CONFIG_PRINTK`` (usually enabled by default) + +**Pros**: + +- No need to learn anything, simple to use +- Easy to modify exactly to your needs (formatting of the data (See: + :doc:`/core-api/printk-formats`), visibility in the log) +- Can cause delays in the execution of the code (beneficial to confirm whether + timing is a factor) + +**Cons**: + +- Requires rebuilding the kernel/module +- Can cause delays in the execution of the code (which can cause issues to be + not reproducible) + +For the full documentation see :doc:`/core-api/printk-basics` + +Trace_printk +~~~~~~~~~~~~ + +Prerequisite: ``CONFIG_DYNAMIC_FTRACE`` & ``#include <linux/ftrace.h>`` + +It is a tiny bit less comfortable to use than printk(), because you will have +to read the messages from the trace file (See: :ref:`read_ftrace_log` +instead of from the kernel log, but very useful when printk() adds unwanted +delays into the code execution, causing issues to be flaky or hidden.) + +If the processing of this still causes timing issues then you can try +trace_puts(). + +For the full Documentation see trace_printk() + +dev_dbg +~~~~~~~ + +Print statement, which can be targeted by +:ref:`process/debugging/userspace_debugging_guide:dynamic debug` that contains +additional information about the device used within the context. + +**When is it appropriate to leave a debug print in the code?** + +Permanent debug statements have to be useful for a developer to troubleshoot +driver misbehavior. Judging that is a bit more of an art than a science, but +some guidelines are in the :ref:`Coding style guidelines +<process/coding-style:13) printing kernel messages>`. In almost all cases the +debug statements shouldn't be upstreamed, as a working driver is supposed to be +silent. + +Custom printk +~~~~~~~~~~~~~ + +Example:: + + #define core_dbg(fmt, arg...) do { \ + if (core_debug) \ + printk(KERN_DEBUG pr_fmt("core: " fmt), ## arg); \ + } while (0) + +**When should you do this?** + +It is better to just use a pr_debug(), which can later be turned on/off with +dynamic debug. Additionally, a lot of drivers activate these prints via a +variable like ``core_debug`` set by a module parameter. However, Module +parameters `are not recommended anymore +<https://lore.kernel.org/all/2024032757-surcharge-grime-d3dd@gregkh>`_. + +Ftrace +------ + +Creating a custom Ftrace tracepoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A tracepoint adds a hook into your code that will be called and logged when the +tracepoint is enabled. This can be used, for example, to trace hitting a +conditional branch or to dump the internal state at specific points of the code +flow during a debugging session. + +Here is a basic description of :ref:`how to implement new tracepoints +<trace/tracepoints:usage>`. + +For the full event tracing documentation see :doc:`/trace/events` + +For the full Ftrace documentation see :doc:`/trace/ftrace` + +DebugFS +------- + +Prerequisite: ``CONFIG_DEBUG_FS` & `#include <linux/debugfs.h>`` + +DebugFS differs from the other approaches of debugging, as it doesn't write +messages to the kernel log nor add traces to the code. Instead it allows the +developer to handle a set of files. +With these files you can either store values of variables or make +register/memory dumps or you can make these files writable and modify +values/settings in the driver. + +Possible use-cases among others: + +- Store register values +- Keep track of variables +- Store errors +- Store settings +- Toggle a setting like debug on/off +- Error injection + +This is especially useful, when the size of a data dump would be hard to digest +as part of the general kernel log (for example when dumping raw bitstream data) +or when you are not interested in all the values all the time, but with the +possibility to inspect them. + +The general idea is: + +- Create a directory during probe (``struct dentry *parent = + debugfs_create_dir("my_driver", NULL);``) +- Create a file (``debugfs_create_u32("my_value", 444, parent, &my_variable);``) + + - In this example the file is found in + ``/sys/kernel/debug/my_driver/my_value`` (with read permissions for + user/group/all) + - any read of the file will return the current contents of the variable + ``my_variable`` + +- Clean up the directory when removing the device + (``debugfs_remove_recursive(parent);``) + +For the full documentation see :doc:`/filesystems/debugfs`. + +KASAN, UBSAN, lockdep and other error checkers +---------------------------------------------- + +KASAN (Kernel Address Sanitizer) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Prerequisite: ``CONFIG_KASAN`` + +KASAN is a dynamic memory error detector that helps to find use-after-free and +out-of-bounds bugs. It uses compile-time instrumentation to check every memory +access. + +For the full documentation see :doc:`/dev-tools/kasan`. + +UBSAN (Undefined Behavior Sanitizer) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Prerequisite: ``CONFIG_UBSAN`` + +UBSAN relies on compiler instrumentation and runtime checks to detect undefined +behavior. It is designed to find a variety of issues, including signed integer +overflow, array index out of bounds, and more. + +For the full documentation see :doc:`/dev-tools/ubsan` + +lockdep (Lock Dependency Validator) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Prerequisite: ``CONFIG_DEBUG_LOCKDEP`` + +lockdep is a runtime lock dependency validator that detects potential deadlocks +and other locking-related issues in the kernel. +It tracks lock acquisitions and releases, building a dependency graph that is +analyzed for potential deadlocks. +lockdep is especially useful for validating the correctness of lock ordering in +the kernel. + +PSI (Pressure stall information tracking) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Prerequisite: ``CONFIG_PSI`` + +PSI is a measurement tool to identify excessive overcommits on hardware +resources, that can cause performance disruptions or even OOM kills. + +device coredump +--------------- + +Prerequisite: ``#include <linux/devcoredump.h>`` + +Provides the infrastructure for a driver to provide arbitrary data to userland. +It is most often used in conjunction with udev or similar userland application +to listen for kernel uevents, which indicate that the dump is ready. Udev has +rules to copy that file somewhere for long-term storage and analysis, as by +default, the data for the dump is automatically cleaned up after 5 minutes. +That data is analyzed with driver-specific tools or GDB. + +You can find an example implementation at: +`drivers/media/platform/qcom/venus/core.c +<https://elixir.bootlin.com/linux/v6.11.6/source/drivers/media/platform/qcom/venus/core.c#L30>`__ + +**Copyright** ©2024 : Collabora diff --git a/Documentation/process/debugging/index.rst b/Documentation/process/debugging/index.rst new file mode 100644 index 000000000000..f6e4a00dfee3 --- /dev/null +++ b/Documentation/process/debugging/index.rst @@ -0,0 +1,78 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================================ +Debugging advice for Linux Kernel developers +============================================ + +general guides +-------------- + +.. toctree:: + :maxdepth: 1 + + driver_development_debugging_guide + userspace_debugging_guide + +.. only:: subproject and html + +subsystem specific guides +------------------------- + +.. toctree:: + :maxdepth: 1 + + media_specific_debugging_guide + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` + +General debugging advice +======================== + +Depending on the issue, a different set of tools is available to track down the +problem or even to realize whether there is one in the first place. + +As a first step you have to figure out what kind of issue you want to debug. +Depending on the answer, your methodology and choice of tools may vary. + +Do I need to debug with limited access? +--------------------------------------- + +Do you have limited access to the machine or are you unable to stop the running +execution? + +In this case your debugging capability depends on built-in debugging support of +provided distribution kernel. +The :doc:`/process/debugging/userspace_debugging_guide` provides a brief +overview over a range of possible debugging tools in that situation. You can +check the capability of your kernel, in most cases, by looking into config file +within the /boot directory. + +Do I have root access to the system? +------------------------------------ + +Are you easily able to replace the module in question or to install a new +kernel? + +In that case your range of available tools is a lot bigger, you can find the +tools in the :doc:`/process/debugging/driver_development_debugging_guide`. + +Is timing a factor? +------------------- + +It is important to understand if the problem you want to debug manifests itself +consistently (i.e. given a set of inputs you always get the same, incorrect +output), or inconsistently. If it manifests itself inconsistently, some timing +factor might be at play. If inserting delays into the code does change the +behavior, then quite likely timing is a factor. + +When timing does alter the outcome of the code execution using a simple +printk() for debugging purposes may not work, a similar alternative is to use +trace_printk() , which logs the debug messages to the trace file instead of the +kernel log. + +**Copyright** ©2024 : Collabora diff --git a/Documentation/process/debugging/media_specific_debugging_guide.rst b/Documentation/process/debugging/media_specific_debugging_guide.rst new file mode 100644 index 000000000000..c5a93bafaf67 --- /dev/null +++ b/Documentation/process/debugging/media_specific_debugging_guide.rst @@ -0,0 +1,180 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================================ +Debugging and tracing in the media subsystem +============================================ + +This document serves as a starting point and lookup for debugging device +drivers in the media subsystem and to debug these drivers from userspace. + +.. contents:: + :depth: 3 + +General debugging advice +------------------------ + +For general advice see the :doc:`general advice document +</process/debugging/index>`. + +The following sections show you some of the available tools. + +dev_debug module parameter +-------------------------- + +Every video device provides a ``dev_debug`` parameter, which allows to get +further insights into the IOCTLs in the background.:: + + # cat /sys/class/video4linux/video3/name + rkvdec + # echo 0xff > /sys/class/video4linux/video3/dev_debug + # dmesg -wH + [...] videodev: v4l2_open: video3: open (0) + [ +0.000036] video3: VIDIOC_QUERYCAP: driver=rkvdec, card=rkvdec, + bus=platform:rkvdec, version=0x00060900, capabilities=0x84204000, + device_caps=0x04204000 + +For the full documentation see :ref:`driver-api/media/v4l2-dev:video device +debugging` + +dev_dbg() / v4l2_dbg() +---------------------- + +Two debug print statements, which are specific for devices and for the v4l2 +subsystem, avoid adding these to your final submission unless they have +long-term value for investigations. + +For a general overview please see the +:ref:`process/debugging/driver_development_debugging_guide:printk() & friends` +guide. + +- Difference between both? + + - v4l2_dbg() utilizes v4l2_printk() under the hood, which further uses + printk() directly, thus it cannot be targeted by dynamic debug + - dev_dbg() can be targeted by dynamic debug + - v4l2_dbg() has a more specific prefix format for the media subsystem, while + dev_dbg only highlights the driver name and the location of the log + +Dynamic debug +------------- + +A method to trim down the debug output to your needs. + +For general advice see the +:ref:`process/debugging/userspace_debugging_guide:dynamic debug` guide. + +Here is one example, that enables all available pr_debug()'s within the file:: + + $ alias ddcmd='echo $* > /proc/dynamic_debug/control' + $ ddcmd '-p; file v4l2-h264.c +p' + $ grep =p /proc/dynamic_debug/control + drivers/media/v4l2-core/v4l2-h264.c:372 [v4l2_h264]print_ref_list_b =p + "ref_pic_list_b%u (cur_poc %u%c) %s" + drivers/media/v4l2-core/v4l2-h264.c:333 [v4l2_h264]print_ref_list_p =p + "ref_pic_list_p (cur_poc %u%c) %s\n" + +Ftrace +------ + +An internal kernel tracer that can trace static predefined events, function +calls, etc. Very useful for debugging problems without changing the kernel and +understanding the behavior of subsystems. + +For general advice see the +:ref:`process/debugging/userspace_debugging_guide:ftrace` guide. + +DebugFS +------- + +This tool allows you to dump or modify internal values of your driver to files +in a custom filesystem. + +For general advice see the +:ref:`process/debugging/driver_development_debugging_guide:debugfs` guide. + +Perf & alternatives +------------------- + +Tools to measure the various stats on a running system to diagnose issues. + +For general advice see the +:ref:`process/debugging/userspace_debugging_guide:perf & alternatives` guide. + +Example for media devices: + +Gather statistics data for a decoding job: (This example is on a RK3399 SoC +with the rkvdec codec driver using the `fluster test suite +<https://github.com/fluendo/fluster>`__):: + + perf stat -d python3 fluster.py run -d GStreamer-H.264-V4L2SL-Gst1.0 -ts + JVT-AVC_V1 -tv AUD_MW_E -j1 + ... + Performance counter stats for 'python3 fluster.py run -d + GStreamer-H.264-V4L2SL-Gst1.0 -ts JVT-AVC_V1 -tv AUD_MW_E -j1 -v': + + 7794.23 msec task-clock:u # 0.697 CPUs utilized + 0 context-switches:u # 0.000 /sec + 0 cpu-migrations:u # 0.000 /sec + 11901 page-faults:u # 1.527 K/sec + 882671556 cycles:u # 0.113 GHz (95.79%) + 711708695 instructions:u # 0.81 insn per cycle (95.79%) + 10581935 branches:u # 1.358 M/sec (15.13%) + 6871144 branch-misses:u # 64.93% of all branches (95.79%) + 281716547 L1-dcache-loads:u # 36.144 M/sec (95.79%) + 9019581 L1-dcache-load-misses:u # 3.20% of all L1-dcache accesses (95.79%) + <not supported> LLC-loads:u + <not supported> LLC-load-misses:u + + 11.180830431 seconds time elapsed + + 1.502318000 seconds user + 6.377221000 seconds sys + +The availability of events and metrics depends on the system you are running. + +Error checking & panic analysis +------------------------------- + +Various Kernel configuration options to enhance error detection of the Linux +Kernel with the cost of lowering performance. + +For general advice see the +:ref:`process/debugging/driver_development_debugging_guide:kasan, ubsan, +lockdep and other error checkers` guide. + +Driver verification with v4l2-compliance +---------------------------------------- + +To verify, that a driver adheres to the v4l2 API, the tool v4l2-compliance is +used, which is part of the `v4l_utils +<https://git.linuxtv.org/v4l-utils.git>`__, a suite of userspace tools to work +with the media subsystem. + +To see the detailed media topology (and check it) use:: + + v4l2-compliance -M /dev/mediaX --verbose + +You can also run a full compliance check for all devices referenced in the +media topology with:: + + v4l2-compliance -m /dev/mediaX + +Debugging problems with receiving video +--------------------------------------- + +Implementing vidioc_log_status in the driver: this can log the current status +to the kernel log. It's called by v4l2-ctl --log-status. Very useful for +debugging problems with receiving video (TV/S-Video/HDMI/etc) since the video +signal is external (so unpredictable). Less useful with camera sensor inputs +since you have control over what the camera sensor does. + +Usually you can just assign the default:: + + .vidioc_log_status = v4l2_ctrl_log_status, + +But you can also create your own callback, to create a custom status log. + +You can find an example in the cobalt driver +(`drivers/media/pci/cobalt/cobalt-v4l2.c <https://elixir.bootlin.com/linux/v6.11.6/source/drivers/media/pci/cobalt/cobalt-v4l2.c#L567>`__). + +**Copyright** ©2024 : Collabora diff --git a/Documentation/process/debugging/userspace_debugging_guide.rst b/Documentation/process/debugging/userspace_debugging_guide.rst new file mode 100644 index 000000000000..db7396261e07 --- /dev/null +++ b/Documentation/process/debugging/userspace_debugging_guide.rst @@ -0,0 +1,280 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +Userspace debugging advice +========================== + +This document provides a brief overview of common tools to debug the Linux +Kernel from userspace. +For debugging advice aimed at driver developers go :doc:`here +</process/debugging/driver_development_debugging_guide>`. +For general debugging advice, see :doc:`general advice document +</process/debugging/index>`. + +.. contents:: + :depth: 3 + +The following sections show you the available tools. + +Dynamic debug +------------- + +Mechanism to filter what ends up in the kernel log by dis-/en-abling log +messages. + +Prerequisite: ``CONFIG_DYNAMIC_DEBUG`` + +Dynamic debug is only able to target: + +- pr_debug() +- dev_dbg() +- print_hex_dump_debug() +- print_hex_dump_bytes() + +Therefore the usability of this tool is, as of now, quite limited as there is +no uniform rule for adding debug prints to the codebase, resulting in a variety +of ways these prints are implemented. + +Also, note that most debug statements are implemented as a variation of +dprintk(), which have to be activated via a parameter in respective module, +dynamic debug is unable to do that step for you. + +Here is one example, that enables all available pr_debug()'s within the file:: + + $ alias ddcmd='echo $* > /proc/dynamic_debug/control' + $ ddcmd '-p; file v4l2-h264.c +p' + $ grep =p /proc/dynamic_debug/control + drivers/media/v4l2-core/v4l2-h264.c:372 [v4l2_h264]print_ref_list_b =p + "ref_pic_list_b%u (cur_poc %u%c) %s" + drivers/media/v4l2-core/v4l2-h264.c:333 [v4l2_h264]print_ref_list_p =p + "ref_pic_list_p (cur_poc %u%c) %s\n" + +**When should you use this over Ftrace ?** + +- When the code contains one of the valid print statements (see above) or when + you have added multiple pr_debug() statements during development +- When timing is not an issue, meaning if multiple pr_debug() statements in + the code won't cause delays +- When you care more about receiving specific log messages than tracing the + pattern of how a function is called + +For the full documentation see :doc:`/admin-guide/dynamic-debug-howto` + +Ftrace +------ + +Prerequisite: ``CONFIG_DYNAMIC_FTRACE`` + +This tool uses the tracefs file system for the control files and output files. +That file system will be mounted as a ``tracing`` directory, which can be found +in either ``/sys/kernel/`` or ``/sys/debug/kernel/``. + +Some of the most important operations for debugging are: + +- You can perform a function trace by adding a function name to the + ``set_ftrace_filter`` file (which accepts any function name found within the + ``available_filter_functions`` file) or you can specifically disable certain + functions by adding their names to the ``set_ftrace_notrace`` file (more info + at: :ref:`trace/ftrace:dynamic ftrace`). +- In order to find out where calls originate from you can activate the + ``func_stack_trace`` option under ``options/func_stack_trace``. +- Tracing the children of a function call and showing the return values are + possible by adding the desired function in the ``set_graph_function`` file + (requires config ``FUNCTION_GRAPH_RETVAL``); more info at + :ref:`trace/ftrace:dynamic ftrace with the function graph tracer`. + +For the full Ftrace documentation see :doc:`/trace/ftrace` + +Or you could also trace for specific events by :ref:`using event tracing +<trace/events:2. using event tracing>`, which can be defined as described here: +:ref:`Creating a custom Ftrace tracepoint +<process/debugging/driver_development_debugging_guide:ftrace>`. + +For the full Ftrace event tracing documentation see :doc:`/trace/events` + +.. _read_ftrace_log: + +Reading the ftrace log +~~~~~~~~~~~~~~~~~~~~~~ + +The ``trace`` file can be read just like any other file (``cat``, ``tail``, +``head``, ``vim``, etc.), the size of the file is limited by the +``buffer_size_kb`` (``echo 1000 > buffer_size_kb``). The +:ref:`trace/ftrace:trace_pipe` will behave similarly to the ``trace`` file, but +whenever you read from the file the content is consumed. + +Kernelshark +~~~~~~~~~~~ + +A GUI interface to visualize the traces as a graph and list view from the +output of the `trace-cmd +<https://git.kernel.org/pub/scm/utils/trace-cmd/trace-cmd.git/>`__ application. + +For the full documentation see `<https://kernelshark.org/Documentation.html>`__ + +Perf & alternatives +------------------- + +The tools mentioned above provide ways to inspect kernel code, results, +variable values, etc. Sometimes you have to find out first where to look and +for those cases, a box of performance tracking tools can help you to frame the +issue. + +Why should you do a performance analysis? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A performance analysis is a good first step when among other reasons: + +- you cannot define the issue +- you do not know where it occurs +- the running system should not be interrupted or it is a remote system, where + you cannot install a new module/kernel + +How to do a simple analysis with linux tools? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For the start of a performance analysis, you can start with the usual tools +like: + +- ``top`` / ``htop`` / ``atop`` (*get an overview of the system load, see + spikes on specific processes*) +- ``mpstat -P ALL`` (*look at the load distribution among CPUs*) +- ``iostat -x`` (*observe input and output devices utilization and performance*) +- ``vmstat`` (*overview of memory usage on the system*) +- ``pidstat`` (*similar to* ``vmstat`` *but per process, to dial it down to the + target*) +- ``strace -tp $PID`` (*once you know the process, you can figure out how it + communicates with the Kernel*) + +These should help to narrow down the areas to look at sufficiently. + +Diving deeper with perf +~~~~~~~~~~~~~~~~~~~~~~~ + +The **perf** tool provides a series of metrics and events to further dial down +on issues. + +Prerequisite: build or install perf on your system + +Gather statistics data for finding all files starting with ``gcc`` in ``/usr``:: + + # perf stat -d find /usr -name 'gcc*' | wc -l + + Performance counter stats for 'find /usr -name gcc*': + + 1277.81 msec task-clock # 0.997 CPUs utilized + 9 context-switches # 7.043 /sec + 1 cpu-migrations # 0.783 /sec + 704 page-faults # 550.943 /sec + 766548897 cycles # 0.600 GHz (97.15%) + 798285467 instructions # 1.04 insn per cycle (97.15%) + 57582731 branches # 45.064 M/sec (2.85%) + 3842573 branch-misses # 6.67% of all branches (97.15%) + 281616097 L1-dcache-loads # 220.390 M/sec (97.15%) + 4220975 L1-dcache-load-misses # 1.50% of all L1-dcache accesses (97.15%) + <not supported> LLC-loads + <not supported> LLC-load-misses + + 1.281746009 seconds time elapsed + + 0.508796000 seconds user + 0.773209000 seconds sys + + + 52 + +The availability of events and metrics depends on the system you are running. + +For the full documentation see +`<https://perf.wiki.kernel.org/index.php/Main_Page>`__ + +Perfetto +~~~~~~~~ + +A set of tools to measure and analyze how well applications and systems perform. +You can use it to: + +* identify bottlenecks +* optimize code +* make software run faster and more efficiently. + +**What is the difference between perfetto and perf?** + +* perf is tool as part of and specialized for the Linux Kernel and has CLI user + interface. +* perfetto cross-platform performance analysis stack, has extended + functionality into userspace and provides a WEB user interface. + +For the full documentation see `<https://perfetto.dev/docs/>`__ + +Kernel panic analysis tools +--------------------------- + + To capture the crash dump please use ``Kdump`` & ``Kexec``. Below you can find + some advice for analysing the data. + + For the full documentation see the :doc:`/admin-guide/kdump/kdump` + + In order to find the corresponding line in the code you can use `faddr2line + <https://elixir.bootlin.com/linux/v6.11.6/source/scripts/faddr2line>`__; note + that you need to enable ``CONFIG_DEBUG_INFO`` for that to work. + + An alternative to using ``faddr2line`` is the use of ``objdump`` (and its + derivatives for the different platforms like ``aarch64-linux-gnu-objdump``). + Take this line as an example: + + ``[ +0.000240] rkvdec_device_run+0x50/0x138 [rockchip_vdec]``. + + We can find the corresponding line of code by executing:: + + aarch64-linux-gnu-objdump -dS drivers/staging/media/rkvdec/rockchip-vdec.ko | grep rkvdec_device_run\>: -A 40 + 0000000000000ac8 <rkvdec_device_run>: + ac8: d503201f nop + acc: d503201f nop + { + ad0: d503233f paciasp + ad4: a9bd7bfd stp x29, x30, [sp, #-48]! + ad8: 910003fd mov x29, sp + adc: a90153f3 stp x19, x20, [sp, #16] + ae0: a9025bf5 stp x21, x22, [sp, #32] + const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc; + ae4: f9411814 ldr x20, [x0, #560] + struct rkvdec_dev *rkvdec = ctx->dev; + ae8: f9418015 ldr x21, [x0, #768] + if (WARN_ON(!desc)) + aec: b4000654 cbz x20, bb4 <rkvdec_device_run+0xec> + ret = pm_runtime_resume_and_get(rkvdec->dev); + af0: f943d2b6 ldr x22, [x21, #1952] + ret = __pm_runtime_resume(dev, RPM_GET_PUT); + af4: aa0003f3 mov x19, x0 + af8: 52800081 mov w1, #0x4 // #4 + afc: aa1603e0 mov x0, x22 + b00: 94000000 bl 0 <__pm_runtime_resume> + if (ret < 0) { + b04: 37f80340 tbnz w0, #31, b6c <rkvdec_device_run+0xa4> + dev_warn(rkvdec->dev, "Not good\n"); + b08: f943d2a0 ldr x0, [x21, #1952] + b0c: 90000001 adrp x1, 0 <rkvdec_try_ctrl-0x8> + b10: 91000021 add x1, x1, #0x0 + b14: 94000000 bl 0 <_dev_warn> + *bad = 1; + b18: d2800001 mov x1, #0x0 // #0 + ... + + Meaning, in this line from the crash dump:: + + [ +0.000240] rkvdec_device_run+0x50/0x138 [rockchip_vdec] + + I can take the ``0x50`` as offset, which I have to add to the base address + of the corresponding function, which I find in this line:: + + 0000000000000ac8 <rkvdec_device_run>: + + The result of ``0xac8 + 0x50 = 0xb18`` + And when I search for that address within the function I get the + following line:: + + *bad = 1; + b18: d2800001 mov x1, #0x0 + +**Copyright** ©2024 : Collabora diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst index 6455eba3ef0c..aa12f2660194 100644 --- a/Documentation/process/index.rst +++ b/Documentation/process/index.rst @@ -72,13 +72,15 @@ beyond). Dealing with bugs ----------------- -Bugs are a fact of life; it is important that we handle them properly. -The documents below describe our policies around the handling of a couple -of special classes of bugs: regressions and security problems. +Bugs are a fact of life; it is important that we handle them properly. The +documents below provide general advice about debugging and describe our +policies around the handling of a couple of special classes of bugs: +regressions and security problems. .. toctree:: :maxdepth: 1 + debugging/index handling-regressions security-bugs cve diff --git a/Documentation/process/license-rules.rst b/Documentation/process/license-rules.rst index 2ef44ada3f11..59a7832df7d0 100644 --- a/Documentation/process/license-rules.rst +++ b/Documentation/process/license-rules.rst @@ -471,14 +471,16 @@ _`MODULE_LICENSE` source files. "Proprietary" The module is under a proprietary license. - This string is solely for proprietary third - party modules and cannot be used for modules - which have their source code in the kernel - tree. Modules tagged that way are tainting - the kernel with the 'P' flag when loaded and - the kernel module loader refuses to link such - modules against symbols which are exported - with EXPORT_SYMBOL_GPL(). + "Proprietary" is to be understood only as + "The license is not compatible to GPLv2". + This string is solely for non-GPL2 compatible + third party modules and cannot be used for + modules which have their source code in the + kernel tree. Modules tagged that way are + tainting the kernel with the 'P' flag when + loaded and the kernel module loader refuses + to link such modules against symbols which + are exported with EXPORT_SYMBOL_GPL(). ============================= ============================================= diff --git a/Documentation/rust/coding-guidelines.rst b/Documentation/rust/coding-guidelines.rst index 329b070a1d47..a2e326b42410 100644 --- a/Documentation/rust/coding-guidelines.rst +++ b/Documentation/rust/coding-guidelines.rst @@ -227,3 +227,149 @@ The equivalent in Rust may look like (ignoring documentation): That is, the equivalent of ``GPIO_LINE_DIRECTION_IN`` would be referred to as ``gpio::LineDirection::In``. In particular, it should not be named ``gpio::gpio_line_direction::GPIO_LINE_DIRECTION_IN``. + + +Lints +----- + +In Rust, it is possible to ``allow`` particular warnings (diagnostics, lints) +locally, making the compiler ignore instances of a given warning within a given +function, module, block, etc. + +It is similar to ``#pragma GCC diagnostic push`` + ``ignored`` + ``pop`` in C +[#]_: + +.. code-block:: c + + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-function" + static void f(void) {} + #pragma GCC diagnostic pop + +.. [#] In this particular case, the kernel's ``__{always,maybe}_unused`` + attributes (C23's ``[[maybe_unused]]``) may be used; however, the example + is meant to reflect the equivalent lint in Rust discussed afterwards. + +But way less verbose: + +.. code-block:: rust + + #[allow(dead_code)] + fn f() {} + +By that virtue, it makes it possible to comfortably enable more diagnostics by +default (i.e. outside ``W=`` levels). In particular, those that may have some +false positives but that are otherwise quite useful to keep enabled to catch +potential mistakes. + +On top of that, Rust provides the ``expect`` attribute which takes this further. +It makes the compiler warn if the warning was not produced. For instance, the +following will ensure that, when ``f()`` is called somewhere, we will have to +remove the attribute: + +.. code-block:: rust + + #[expect(dead_code)] + fn f() {} + +If we do not, we get a warning from the compiler:: + + warning: this lint expectation is unfulfilled + --> x.rs:3:10 + | + 3 | #[expect(dead_code)] + | ^^^^^^^^^ + | + = note: `#[warn(unfulfilled_lint_expectations)]` on by default + +This means that ``expect``\ s do not get forgotten when they are not needed, which +may happen in several situations, e.g.: + +- Temporary attributes added while developing. + +- Improvements in lints in the compiler, Clippy or custom tools which may + remove a false positive. + +- When the lint is not needed anymore because it was expected that it would be + removed at some point, such as the ``dead_code`` example above. + +It also increases the visibility of the remaining ``allow``\ s and reduces the +chance of misapplying one. + +Thus prefer ``expect`` over ``allow`` unless: + +- Conditional compilation triggers the warning in some cases but not others. + + If there are only a few cases where the warning triggers (or does not + trigger) compared to the total number of cases, then one may consider using + a conditional ``expect`` (i.e. ``cfg_attr(..., expect(...))``). Otherwise, + it is likely simpler to just use ``allow``. + +- Inside macros, when the different invocations may create expanded code that + triggers the warning in some cases but not in others. + +- When code may trigger a warning for some architectures but not others, such + as an ``as`` cast to a C FFI type. + +As a more developed example, consider for instance this program: + +.. code-block:: rust + + fn g() {} + + fn main() { + #[cfg(CONFIG_X)] + g(); + } + +Here, function ``g()`` is dead code if ``CONFIG_X`` is not set. Can we use +``expect`` here? + +.. code-block:: rust + + #[expect(dead_code)] + fn g() {} + + fn main() { + #[cfg(CONFIG_X)] + g(); + } + +This would emit a lint if ``CONFIG_X`` is set, since it is not dead code in that +configuration. Therefore, in cases like this, we cannot use ``expect`` as-is. + +A simple possibility is using ``allow``: + +.. code-block:: rust + + #[allow(dead_code)] + fn g() {} + + fn main() { + #[cfg(CONFIG_X)] + g(); + } + +An alternative would be using a conditional ``expect``: + +.. code-block:: rust + + #[cfg_attr(not(CONFIG_X), expect(dead_code))] + fn g() {} + + fn main() { + #[cfg(CONFIG_X)] + g(); + } + +This would ensure that, if someone introduces another call to ``g()`` somewhere +(e.g. unconditionally), then it would be spotted that it is not dead code +anymore. However, the ``cfg_attr`` is more complex than a simple ``allow``. + +Therefore, it is likely that it is not worth using conditional ``expect``\ s when +more than one or two configurations are involved or when the lint may be +triggered due to non-local changes (such as ``dead_code``). + +For more information about diagnostics in Rust, please see: + + https://doc.rust-lang.org/stable/reference/attributes/diagnostics.html diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst index 2d107982c87b..4aa50e5fcb8c 100644 --- a/Documentation/rust/quick-start.rst +++ b/Documentation/rust/quick-start.rst @@ -87,6 +87,23 @@ they should generally work out of the box, e.g.:: zypper install rust rust1.79-src rust-bindgen clang +Ubuntu +****** + +Ubuntu LTS and non-LTS (interim) releases provide recent Rust releases and thus +they should generally work out of the box, e.g.:: + + apt install rustc-1.80 rust-1.80-src bindgen-0.65 rustfmt-1.80 rust-1.80-clippy + +``RUST_LIB_SRC`` needs to be set when using the versioned packages, e.g.:: + + RUST_LIB_SRC=/usr/src/rustc-$(rustc-1.80 --version | cut -d' ' -f2)/library + +In addition, ``bindgen-0.65`` is available in newer releases (24.04 LTS and +24.10), but it may not be available in older ones (20.04 LTS and 22.04 LTS), +thus ``bindgen`` may need to be built manually (please see below). + + Requirements: Building ---------------------- diff --git a/MAINTAINERS b/MAINTAINERS index 550a74e111ef..85cdc618a51c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12436,6 +12436,7 @@ F: include/trace/misc/sunrpc.h F: include/uapi/linux/nfsd/ F: include/uapi/linux/sunrpc/ F: net/sunrpc/ +F: tools/net/sunrpc/ KERNEL PACMAN PACKAGING (in addition to generic KERNEL BUILD) M: Thomas Weißschuh <linux@weissschuh.net> @@ -13926,6 +13927,12 @@ R: schalla@marvell.com R: vattunuru@marvell.com F: drivers/vdpa/octeon_ep/ +MARVELL OCTEON HOTPLUG DRIVER +R: Shijith Thotton <sthotton@marvell.com> +R: Vamsi Attunuru <vattunuru@marvell.com> +S: Supported +F: drivers/pci/hotplug/octep_hp.c + MATROX FRAMEBUFFER DRIVER L: linux-fbdev@vger.kernel.org S: Orphan @@ -17993,8 +18000,8 @@ M: Bartosz Golaszewski <brgl@bgdev.pl> L: linux-pci@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git -F: drivers/pci/pwrctl/* -F: include/linux/pci-pwrctl.h +F: drivers/pci/pwrctrl/* +F: include/linux/pci-pwrctrl.h PCI SUBSYSTEM M: Bjorn Helgaas <bhelgaas@google.com> @@ -18016,6 +18023,15 @@ F: include/linux/of_pci.h F: include/linux/pci* F: include/uapi/linux/pci* +PCIE BANDWIDTH CONTROLLER +M: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com> +L: linux-pci@vger.kernel.org +S: Supported +F: drivers/pci/pcie/bwctrl.c +F: drivers/thermal/pcie_cooling.c +F: include/linux/pci-bwctrl.h +F: tools/testing/selftests/pcie_bwctrl/ + PCIE DRIVER FOR AMAZON ANNAPURNA LABS M: Jonathan Chocron <jonnyc@amazon.com> L: linux-pci@vger.kernel.org @@ -20367,6 +20383,7 @@ B: https://github.com/Rust-for-Linux/linux/issues C: zulip://rust-for-linux.zulipchat.com P: https://rust-for-linux.com/contributing T: git https://github.com/Rust-for-Linux/linux.git rust-next +F: .clippy.toml F: Documentation/rust/ F: include/trace/events/rust_sample.h F: rust/ @@ -20375,6 +20392,13 @@ F: scripts/*rust* F: tools/testing/selftests/rust/ K: \b(?i:rust)\b +RUST [ALLOC] +M: Danilo Krummrich <dakr@kernel.org> +L: rust-for-linux@vger.kernel.org +S: Maintained +F: rust/kernel/alloc.rs +F: rust/kernel/alloc/ + RXRPC SOCKETS (AF_RXRPC) M: David Howells <dhowells@redhat.com> M: Marc Dionne <marc.dionne@auristor.com> @@ -446,18 +446,23 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS) export rust_common_flags := --edition=2021 \ -Zbinary_dep_depinfo=y \ -Astable_features \ - -Dunsafe_op_in_unsafe_fn \ -Dnon_ascii_idents \ + -Dunsafe_op_in_unsafe_fn \ + -Wmissing_docs \ -Wrust_2018_idioms \ -Wunreachable_pub \ - -Wmissing_docs \ - -Wrustdoc::missing_crate_level_docs \ -Wclippy::all \ + -Wclippy::ignored_unit_patterns \ -Wclippy::mut_mut \ -Wclippy::needless_bitwise_bool \ -Wclippy::needless_continue \ + -Aclippy::needless_lifetimes \ -Wclippy::no_mangle_with_rust_abi \ - -Wclippy::dbg_macro + -Wclippy::undocumented_unsafe_blocks \ + -Wclippy::unnecessary_safety_comment \ + -Wclippy::unnecessary_safety_doc \ + -Wrustdoc::missing_crate_level_docs \ + -Wrustdoc::unescaped_backticks KBUILD_HOSTCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(HOST_LFS_CFLAGS) \ $(HOSTCFLAGS) -I $(srctree)/scripts/include @@ -582,6 +587,9 @@ endif # Allows the usage of unstable features in stable compilers. export RUSTC_BOOTSTRAP := 1 +# Allows finding `.clippy.toml` in out-of-srctree builds. +export CLIPPY_CONF_DIR := $(srctree) + export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN export HOSTRUSTC KBUILD_HOSTRUSTFLAGS diff --git a/arch/m68k/coldfire/device.c b/arch/m68k/coldfire/device.c index 7dab46728aed..b6958ec2a220 100644 --- a/arch/m68k/coldfire/device.c +++ b/arch/m68k/coldfire/device.c @@ -93,7 +93,7 @@ static struct platform_device mcf_uart = { .dev.platform_data = mcf_uart_platform_data, }; -#if IS_ENABLED(CONFIG_FEC) +#ifdef MCFFEC_BASE0 #ifdef CONFIG_M5441x #define FEC_NAME "enet-fec" @@ -145,6 +145,7 @@ static struct platform_device mcf_fec0 = { .platform_data = FEC_PDATA, } }; +#endif /* MCFFEC_BASE0 */ #ifdef MCFFEC_BASE1 static struct resource mcf_fec1_resources[] = { @@ -182,7 +183,6 @@ static struct platform_device mcf_fec1 = { } }; #endif /* MCFFEC_BASE1 */ -#endif /* CONFIG_FEC */ #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) /* @@ -624,12 +624,12 @@ static struct platform_device mcf_flexcan0 = { static struct platform_device *mcf_devices[] __initdata = { &mcf_uart, -#if IS_ENABLED(CONFIG_FEC) +#ifdef MCFFEC_BASE0 &mcf_fec0, +#endif #ifdef MCFFEC_BASE1 &mcf_fec1, #endif -#endif #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) &mcf_qspi, #endif diff --git a/arch/m68k/include/asm/mcfgpio.h b/arch/m68k/include/asm/mcfgpio.h index 019f24439546..9c91ecdafc45 100644 --- a/arch/m68k/include/asm/mcfgpio.h +++ b/arch/m68k/include/asm/mcfgpio.h @@ -136,7 +136,7 @@ static inline void gpio_free(unsigned gpio) * read-modify-write as well as those controlled by the EPORT and GPIO modules. */ #define MCFGPIO_SCR_START 40 -#elif defined(CONFIGM5441x) +#elif defined(CONFIG_M5441x) /* The m5441x EPORT doesn't have its own GPIO port, uses PORT C */ #define MCFGPIO_SCR_START 0 #else diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c index 3e34b4473d3a..10fd5b3e63e7 100644 --- a/arch/parisc/kernel/ftrace.c +++ b/arch/parisc/kernel/ftrace.c @@ -87,7 +87,7 @@ int ftrace_enable_ftrace_graph_caller(void) int ftrace_disable_ftrace_graph_caller(void) { - static_key_enable(&ftrace_graph_enable.key); + static_key_disable(&ftrace_graph_enable.key); return 0; } #endif diff --git a/arch/s390/pci/pci_bus.c b/arch/s390/pci/pci_bus.c index 1b74a000ff64..d5ace00d10f0 100644 --- a/arch/s390/pci/pci_bus.c +++ b/arch/s390/pci/pci_bus.c @@ -53,7 +53,7 @@ static int zpci_bus_prepare_device(struct zpci_dev *zdev) zpci_setup_bus_resources(zdev); for (i = 0; i < PCI_STD_NUM_BARS; i++) { if (zdev->bars[i].res) - pci_bus_add_resource(zdev->zbus->bus, zdev->bars[i].res, 0); + pci_bus_add_resource(zdev->zbus->bus, zdev->bars[i].res); } } diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c index 55c4b07ec1f6..0c316bae1726 100644 --- a/arch/x86/pci/acpi.c +++ b/arch/x86/pci/acpi.c @@ -250,6 +250,125 @@ void __init pci_acpi_crs_quirks(void) pr_info("Please notify linux-pci@vger.kernel.org so future kernels can do this automatically\n"); } +/* + * Check if pdev is part of a PCIe switch that is directly below the + * specified bridge. + */ +static bool pcie_switch_directly_under(struct pci_dev *bridge, + struct pci_dev *pdev) +{ + struct pci_dev *parent = pci_upstream_bridge(pdev); + + /* If the device doesn't have a parent, it's not under anything */ + if (!parent) + return false; + + /* + * If the device has a PCIe type, check if it is below the + * corresponding PCIe switch components (if applicable). Then check + * if its upstream port is directly beneath the specified bridge. + */ + switch (pci_pcie_type(pdev)) { + case PCI_EXP_TYPE_UPSTREAM: + return parent == bridge; + + case PCI_EXP_TYPE_DOWNSTREAM: + if (pci_pcie_type(parent) != PCI_EXP_TYPE_UPSTREAM) + return false; + parent = pci_upstream_bridge(parent); + return parent == bridge; + + case PCI_EXP_TYPE_ENDPOINT: + if (pci_pcie_type(parent) != PCI_EXP_TYPE_DOWNSTREAM) + return false; + parent = pci_upstream_bridge(parent); + if (!parent || pci_pcie_type(parent) != PCI_EXP_TYPE_UPSTREAM) + return false; + parent = pci_upstream_bridge(parent); + return parent == bridge; + } + + return false; +} + +static bool pcie_has_usb4_host_interface(struct pci_dev *pdev) +{ + struct fwnode_handle *fwnode; + + /* + * For USB4, the tunneled PCIe Root or Downstream Ports are marked + * with the "usb4-host-interface" ACPI property, so we look for + * that first. This should cover most cases. + */ + fwnode = fwnode_find_reference(dev_fwnode(&pdev->dev), + "usb4-host-interface", 0); + if (!IS_ERR(fwnode)) { + fwnode_handle_put(fwnode); + return true; + } + + /* + * Any integrated Thunderbolt 3/4 PCIe Root Ports from Intel + * before Alder Lake do not have the "usb4-host-interface" + * property so we use their PCI IDs instead. All these are + * tunneled. This list is not expected to grow. + */ + if (pdev->vendor == PCI_VENDOR_ID_INTEL) { + switch (pdev->device) { + /* Ice Lake Thunderbolt 3 PCIe Root Ports */ + case 0x8a1d: + case 0x8a1f: + case 0x8a21: + case 0x8a23: + /* Tiger Lake-LP Thunderbolt 4 PCIe Root Ports */ + case 0x9a23: + case 0x9a25: + case 0x9a27: + case 0x9a29: + /* Tiger Lake-H Thunderbolt 4 PCIe Root Ports */ + case 0x9a2b: + case 0x9a2d: + case 0x9a2f: + case 0x9a31: + return true; + } + } + + return false; +} + +bool arch_pci_dev_is_removable(struct pci_dev *pdev) +{ + struct pci_dev *parent, *root; + + /* pdev without a parent or Root Port is never tunneled */ + parent = pci_upstream_bridge(pdev); + if (!parent) + return false; + root = pcie_find_root_port(pdev); + if (!root) + return false; + + /* Internal PCIe devices are not tunneled */ + if (!root->external_facing) + return false; + + /* Anything directly behind a "usb4-host-interface" is tunneled */ + if (pcie_has_usb4_host_interface(parent)) + return true; + + /* + * Check if this is a discrete Thunderbolt/USB4 controller that is + * directly behind the non-USB4 PCIe Root Port marked as + * "ExternalFacingPort". Those are not behind a PCIe tunnel. + */ + if (pcie_switch_directly_under(root, pdev)) + return false; + + /* PCIe devices after the discrete chip are tunneled */ + return true; +} + #ifdef CONFIG_PCI_MMCONFIG static int check_segment(u16 seg, struct device *dev, char *estr) { diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index 98a9bb92d75c..0681ecfe3430 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -757,7 +757,7 @@ static void pci_amd_enable_64bit_bar(struct pci_dev *dev) dev_info(&dev->dev, "adding root bus resource %pR (tainting kernel)\n", res); add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); - pci_bus_add_resource(dev->bus, res, 0); + pci_bus_add_resource(dev->bus, res); } base = ((res->start >> 8) & AMD_141b_MMIO_BASE_MMIOBASE_MASK) | diff --git a/drivers/ata/acard-ahci.c b/drivers/ata/acard-ahci.c index 547f56341705..3999305b5356 100644 --- a/drivers/ata/acard-ahci.c +++ b/drivers/ata/acard-ahci.c @@ -370,7 +370,7 @@ static int acard_ahci_init_one(struct pci_dev *pdev, const struct pci_device_id /* AHCI controllers often implement SFF compatible interface. * Grab all PCI BARs just in case. */ - rc = pcim_iomap_regions_request_all(pdev, 1 << AHCI_PCI_BAR, DRV_NAME); + rc = pcim_request_all_regions(pdev, DRV_NAME); if (rc == -EBUSY) pcim_pin_device(pdev); if (rc) @@ -386,7 +386,9 @@ static int acard_ahci_init_one(struct pci_dev *pdev, const struct pci_device_id if (!(hpriv->flags & AHCI_HFLAG_NO_MSI)) pci_enable_msi(pdev); - hpriv->mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR]; + hpriv->mmio = pcim_iomap(pdev, AHCI_PCI_BAR, 0); + if (!hpriv->mmio) + return -ENOMEM; /* save initial config */ ahci_save_initial_config(&pdev->dev, hpriv); diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 2d3d3d67b4d9..8d27c567be1c 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1869,7 +1869,7 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) /* AHCI controllers often implement SFF compatible interface. * Grab all PCI BARs just in case. */ - rc = pcim_iomap_regions_request_all(pdev, 1 << ahci_pci_bar, DRV_NAME); + rc = pcim_request_all_regions(pdev, DRV_NAME); if (rc == -EBUSY) pcim_pin_device(pdev); if (rc) @@ -1893,7 +1893,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (ahci_sb600_enable_64bit(pdev)) hpriv->flags &= ~AHCI_HFLAG_32BIT_ONLY; - hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar]; + hpriv->mmio = pcim_iomap(pdev, ahci_pci_bar, 0); + if (!hpriv->mmio) + return -ENOMEM; /* detect remapped nvme devices */ ahci_remap_check(pdev, ahci_pci_bar, hpriv); diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs index b0227cf9ddd3..5de7223beb4d 100644 --- a/drivers/block/rnull.rs +++ b/drivers/block/rnull.rs @@ -32,7 +32,7 @@ module! { } struct NullBlkModule { - _disk: Pin<Box<Mutex<GenDisk<NullBlkDevice>>>>, + _disk: Pin<KBox<Mutex<GenDisk<NullBlkDevice>>>>, } impl kernel::Module for NullBlkModule { @@ -47,7 +47,7 @@ impl kernel::Module for NullBlkModule { .rotational(false) .build(format_args!("rnullb{}", 0), tagset)?; - let disk = Box::pin_init(new_mutex!(disk, "nullb:disk"), flags::GFP_KERNEL)?; + let disk = KBox::pin_init(new_mutex!(disk, "nullb:disk"), flags::GFP_KERNEL)?; Ok(Self { _disk: disk }) } diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index f49818a13013..788a11cdb34b 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -129,16 +129,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; - ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + ret = pcim_request_all_regions(pdev, pci_name(pdev)); if (ret) { - dev_err(&pdev->dev, "Failed to map pci regions.\n"); + dev_err(&pdev->dev, "Failed to request PCI regions.\n"); goto out_err; } i = 0; for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { bar = &accel_pci_dev->pci_bars[i++]; - bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to ioremap PCI region.\n"); + ret = -ENOMEM; + goto out_err; + } } pci_set_master(pdev); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 659905e45950..115eabfd1f6b 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -131,16 +131,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; - ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + ret = pcim_request_all_regions(pdev, pci_name(pdev)); if (ret) { - dev_err(&pdev->dev, "Failed to map pci regions.\n"); + dev_err(&pdev->dev, "Failed to request PCI regions.\n"); goto out_err; } i = 0; for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { bar = &accel_pci_dev->pci_bars[i++]; - bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + bar->virt_addr = pcim_iomap(pdev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to ioremap PCI region.\n"); + ret = -ENOMEM; + goto out_err; + } } pci_set_master(pdev); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 400e36d9908f..94d0e73e42de 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -739,18 +739,22 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, dev_err(dev, "Unable to get usable DMA configuration\n"); goto clear_drvdata; } - /* Map PF's configuration registers */ - err = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, - OTX2_CPT_DRV_NAME); + err = pcim_request_all_regions(pdev, OTX2_CPT_DRV_NAME); if (err) { - dev_err(dev, "Couldn't get PCI resources 0x%x\n", err); + dev_err(dev, "Couldn't request PCI resources 0x%x\n", err); goto clear_drvdata; } pci_set_master(pdev); pci_set_drvdata(pdev, cptpf); cptpf->pdev = pdev; - cptpf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + /* Map PF's configuration registers */ + cptpf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); + if (!cptpf->reg_base) { + err = -ENOMEM; + dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", err); + goto clear_drvdata; + } /* Check if AF driver is up, otherwise defer probe */ err = cpt_is_pf_usable(cptpf); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index 527d34cc258b..d0b6ee901f62 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -358,9 +358,8 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, dev_err(dev, "Unable to get usable DMA configuration\n"); goto clear_drvdata; } - /* Map VF's configuration registers */ - ret = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, - OTX2_CPTVF_DRV_NAME); + + ret = pcim_request_all_regions(pdev, OTX2_CPTVF_DRV_NAME); if (ret) { dev_err(dev, "Couldn't get PCI resources 0x%x\n", ret); goto clear_drvdata; @@ -369,7 +368,13 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, cptvf); cptvf->pdev = pdev; - cptvf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + /* Map VF's configuration registers */ + cptvf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); + if (!cptvf->reg_base) { + ret = -ENOMEM; + dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", ret); + goto clear_drvdata; + } otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag); diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index 80cac3a5f976..602807d6afcc 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -39,14 +39,6 @@ struct cci_drvdata { struct dfl_fpga_cdev *cdev; /* container device */ }; -static void __iomem *cci_pci_ioremap_bar0(struct pci_dev *pcidev) -{ - if (pcim_iomap_regions(pcidev, BIT(0), DRV_NAME)) - return NULL; - - return pcim_iomap_table(pcidev)[0]; -} - static int cci_pci_alloc_irq(struct pci_dev *pcidev) { int ret, nvec = pci_msix_vec_count(pcidev); @@ -235,9 +227,9 @@ static int find_dfls_by_default(struct pci_dev *pcidev, u64 v; /* start to find Device Feature List from Bar 0 */ - base = cci_pci_ioremap_bar0(pcidev); - if (!base) - return -ENOMEM; + base = pcim_iomap_region(pcidev, 0, DRV_NAME); + if (IS_ERR(base)) + return PTR_ERR(base); /* * PF device has FME and Ports/AFUs, and VF device only has one @@ -296,7 +288,7 @@ static int find_dfls_by_default(struct pci_dev *pcidev, } /* release I/O mappings for next step enumeration */ - pcim_iounmap_regions(pcidev, BIT(0)); + pcim_iounmap_region(pcidev, 0); return ret; } diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c index 421d7e3a6c66..cd20604f26de 100644 --- a/drivers/gpio/gpio-merrifield.c +++ b/drivers/gpio/gpio-merrifield.c @@ -78,24 +78,25 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id if (retval) return retval; - retval = pcim_iomap_regions(pdev, BIT(1) | BIT(0), pci_name(pdev)); - if (retval) - return dev_err_probe(dev, retval, "I/O memory mapping error\n"); - - base = pcim_iomap_table(pdev)[1]; + base = pcim_iomap_region(pdev, 1, pci_name(pdev)); + if (IS_ERR(base)) + return dev_err_probe(dev, PTR_ERR(base), "I/O memory mapping error\n"); irq_base = readl(base + 0 * sizeof(u32)); gpio_base = readl(base + 1 * sizeof(u32)); /* Release the IO mapping, since we already get the info from BAR1 */ - pcim_iounmap_regions(pdev, BIT(1)); + pcim_iounmap_region(pdev, 1); priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; priv->dev = dev; - priv->reg_base = pcim_iomap_table(pdev)[0]; + priv->reg_base = pcim_iomap_region(pdev, 0, pci_name(pdev)); + if (IS_ERR(priv->reg_base)) + return dev_err_probe(dev, PTR_ERR(priv->reg_base), + "I/O memory mapping error\n"); priv->pin_info.pin_ranges = mrfld_gpio_ranges; priv->pin_info.nranges = ARRAY_SIZE(mrfld_gpio_ranges); diff --git a/drivers/gpu/drm/drm_panic_qr.rs b/drivers/gpu/drm/drm_panic_qr.rs index 1ef56cb07dfb..09500cddc009 100644 --- a/drivers/gpu/drm/drm_panic_qr.rs +++ b/drivers/gpu/drm/drm_panic_qr.rs @@ -209,12 +209,9 @@ const FORMAT_INFOS_QR_L: [u16; 8] = [ impl Version { /// Returns the smallest QR version than can hold these segments. fn from_segments(segments: &[&Segment<'_>]) -> Option<Version> { - for v in (1..=40).map(|k| Version(k)) { - if v.max_data() * 8 >= segments.iter().map(|s| s.total_size_bits(v)).sum() { - return Some(v); - } - } - None + (1..=40) + .map(Version) + .find(|&v| v.max_data() * 8 >= segments.iter().map(|s| s.total_size_bits(v)).sum()) } fn width(&self) -> u8 { @@ -242,7 +239,7 @@ impl Version { } fn alignment_pattern(&self) -> &'static [u8] { - &ALIGNMENT_PATTERNS[self.0 - 1] + ALIGNMENT_PATTERNS[self.0 - 1] } fn poly(&self) -> &'static [u8] { @@ -479,7 +476,7 @@ struct EncodedMsg<'a> { /// Data to be put in the QR code, with correct segment encoding, padding, and /// Error Code Correction. impl EncodedMsg<'_> { - fn new<'a, 'b>(segments: &[&Segment<'b>], data: &'a mut [u8]) -> Option<EncodedMsg<'a>> { + fn new<'a>(segments: &[&Segment<'_>], data: &'a mut [u8]) -> Option<EncodedMsg<'a>> { let version = Version::from_segments(segments)?; let ec_size = version.ec_size(); let g1_blocks = version.g1_blocks(); @@ -492,7 +489,7 @@ impl EncodedMsg<'_> { data.fill(0); let mut em = EncodedMsg { - data: data, + data, ec_size, g1_blocks, g2_blocks, @@ -722,7 +719,10 @@ impl QrImage<'_> { fn is_finder(&self, x: u8, y: u8) -> bool { let end = self.width - 8; - (x < 8 && y < 8) || (x < 8 && y >= end) || (x >= end && y < 8) + #[expect(clippy::nonminimal_bool)] + { + (x < 8 && y < 8) || (x < 8 && y >= end) || (x >= end && y < 8) + } } // Alignment pattern: 5x5 squares in a grid. @@ -979,10 +979,11 @@ pub unsafe extern "C" fn drm_panic_qr_generate( /// * `url_len`: Length of the URL. /// /// * If `url_len` > 0, remove the 2 segments header/length and also count the -/// conversion to numeric segments. +/// conversion to numeric segments. /// * If `url_len` = 0, only removes 3 bytes for 1 binary segment. #[no_mangle] pub extern "C" fn drm_panic_qr_max_data_size(version: u8, url_len: usize) -> usize { + #[expect(clippy::manual_range_contains)] if version < 1 || version > 40 { return 0; } diff --git a/drivers/hwmon/aquacomputer_d5next.c b/drivers/hwmon/aquacomputer_d5next.c index 34cac27e4dde..0dcb8a3a691d 100644 --- a/drivers/hwmon/aquacomputer_d5next.c +++ b/drivers/hwmon/aquacomputer_d5next.c @@ -597,7 +597,7 @@ struct aqc_data { /* Sensor values */ s32 temp_input[20]; /* Max 4 physical and 16 virtual or 8 physical and 12 virtual */ - s32 speed_input[8]; + s32 speed_input[9]; u32 speed_input_min[1]; u32 speed_input_target[1]; u32 speed_input_max[1]; diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c index 1f36af2cd2d9..fbe673009126 100644 --- a/drivers/hwmon/tmp108.c +++ b/drivers/hwmon/tmp108.c @@ -452,12 +452,7 @@ static int p3t1085_i3c_probe(struct i3c_device *i3cdev) struct device *dev = i3cdev_to_dev(i3cdev); struct regmap *regmap; -#ifdef CONFIG_REGMAP_I3C regmap = devm_regmap_init_i3c(i3cdev, &tmp108_regmap_config); -#else - regmap = ERR_PTR(-ENODEV); -#endif - if (IS_ERR(regmap)) return dev_err_probe(dev, PTR_ERR(regmap), "Failed to register i3c regmap\n"); diff --git a/drivers/hwmon/tps23861.c b/drivers/hwmon/tps23861.c index dfcfb09d9f3c..80fb03f30c30 100644 --- a/drivers/hwmon/tps23861.c +++ b/drivers/hwmon/tps23861.c @@ -132,7 +132,7 @@ static int tps23861_read_temp(struct tps23861_data *data, long *val) if (err < 0) return err; - *val = (regval * TEMPERATURE_LSB) - 20000; + *val = ((long)regval * TEMPERATURE_LSB) - 20000; return 0; } diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c index 0d7b9839e5b6..e9d8d28e055f 100644 --- a/drivers/hwtracing/intel_th/pci.c +++ b/drivers/hwtracing/intel_th/pci.c @@ -23,7 +23,6 @@ enum { TH_PCI_RTIT_BAR = 4, }; -#define BAR_MASK (BIT(TH_PCI_CONFIG_BAR) | BIT(TH_PCI_STH_SW_BAR)) #define PCI_REG_NPKDSC 0x80 #define NPKDSC_TSACT BIT(5) @@ -83,10 +82,16 @@ static int intel_th_pci_probe(struct pci_dev *pdev, if (err) return err; - err = pcim_iomap_regions_request_all(pdev, BAR_MASK, DRIVER_NAME); + err = pcim_request_all_regions(pdev, DRIVER_NAME); if (err) return err; + if (!pcim_iomap(pdev, TH_PCI_CONFIG_BAR, 0)) + return -ENOMEM; + + if (!pcim_iomap(pdev, TH_PCI_STH_SW_BAR, 0)) + return -ENOMEM; + if (pdev->resource[TH_PCI_RTIT_BAR].start) { resource[TH_MMIO_RTIT] = pdev->resource[TH_PCI_RTIT_BAR]; r++; diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 6f3eb710a75d..42310c9a00c2 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -282,7 +282,8 @@ static int i3c_device_uevent(const struct device *dev, struct kobj_uevent_env *e struct i3c_device_info devinfo; u16 manuf, part, ext; - i3c_device_get_info(i3cdev, &devinfo); + if (i3cdev->desc) + devinfo = i3cdev->desc->info; manuf = I3C_PID_MANUF_ID(devinfo.pid); part = I3C_PID_PART_ID(devinfo.pid); ext = I3C_PID_EXTRA_INFO(devinfo.pid); @@ -345,10 +346,10 @@ const struct bus_type i3c_bus_type = { EXPORT_SYMBOL_GPL(i3c_bus_type); static enum i3c_addr_slot_status -i3c_bus_get_addr_slot_status(struct i3c_bus *bus, u16 addr) +i3c_bus_get_addr_slot_status_mask(struct i3c_bus *bus, u16 addr, u32 mask) { unsigned long status; - int bitpos = addr * 2; + int bitpos = addr * I3C_ADDR_SLOT_STATUS_BITS; if (addr > I2C_MAX_ADDR) return I3C_ADDR_SLOT_RSVD; @@ -356,22 +357,33 @@ i3c_bus_get_addr_slot_status(struct i3c_bus *bus, u16 addr) status = bus->addrslots[bitpos / BITS_PER_LONG]; status >>= bitpos % BITS_PER_LONG; - return status & I3C_ADDR_SLOT_STATUS_MASK; + return status & mask; } -static void i3c_bus_set_addr_slot_status(struct i3c_bus *bus, u16 addr, - enum i3c_addr_slot_status status) +static enum i3c_addr_slot_status +i3c_bus_get_addr_slot_status(struct i3c_bus *bus, u16 addr) +{ + return i3c_bus_get_addr_slot_status_mask(bus, addr, I3C_ADDR_SLOT_STATUS_MASK); +} + +static void i3c_bus_set_addr_slot_status_mask(struct i3c_bus *bus, u16 addr, + enum i3c_addr_slot_status status, u32 mask) { - int bitpos = addr * 2; + int bitpos = addr * I3C_ADDR_SLOT_STATUS_BITS; unsigned long *ptr; if (addr > I2C_MAX_ADDR) return; ptr = bus->addrslots + (bitpos / BITS_PER_LONG); - *ptr &= ~((unsigned long)I3C_ADDR_SLOT_STATUS_MASK << - (bitpos % BITS_PER_LONG)); - *ptr |= (unsigned long)status << (bitpos % BITS_PER_LONG); + *ptr &= ~((unsigned long)mask << (bitpos % BITS_PER_LONG)); + *ptr |= ((unsigned long)status & mask) << (bitpos % BITS_PER_LONG); +} + +static void i3c_bus_set_addr_slot_status(struct i3c_bus *bus, u16 addr, + enum i3c_addr_slot_status status) +{ + i3c_bus_set_addr_slot_status_mask(bus, addr, status, I3C_ADDR_SLOT_STATUS_MASK); } static bool i3c_bus_dev_addr_is_avail(struct i3c_bus *bus, u8 addr) @@ -383,13 +395,44 @@ static bool i3c_bus_dev_addr_is_avail(struct i3c_bus *bus, u8 addr) return status == I3C_ADDR_SLOT_FREE; } +/* + * ┌────┬─────────────┬───┬─────────┬───┐ + * │S/Sr│ 7'h7E RnW=0 │ACK│ ENTDAA │ T ├────┐ + * └────┴─────────────┴───┴─────────┴───┘ │ + * ┌─────────────────────────────────────────┘ + * │ ┌──┬─────────────┬───┬─────────────────┬────────────────┬───┬─────────┐ + * └─►│Sr│7'h7E RnW=1 │ACK│48bit UID BCR DCR│Assign 7bit Addr│PAR│ ACK/NACK│ + * └──┴─────────────┴───┴─────────────────┴────────────────┴───┴─────────┘ + * Some master controllers (such as HCI) need to prepare the entire above transaction before + * sending it out to the I3C bus. This means that a 7-bit dynamic address needs to be allocated + * before knowing the target device's UID information. + * + * However, some I3C targets may request specific addresses (called as "init_dyn_addr"), which is + * typically specified by the DT-'s assigned-address property. Lower addresses having higher IBI + * priority. If it is available, i3c_bus_get_free_addr() preferably return a free address that is + * not in the list of desired addresses (called as "init_dyn_addr"). This allows the device with + * the "init_dyn_addr" to switch to its "init_dyn_addr" when it hot-joins the I3C bus. Otherwise, + * if the "init_dyn_addr" is already in use by another I3C device, the target device will not be + * able to switch to its desired address. + * + * If the previous step fails, fallback returning one of the remaining unassigned address, + * regardless of its state in the desired list. + */ static int i3c_bus_get_free_addr(struct i3c_bus *bus, u8 start_addr) { enum i3c_addr_slot_status status; u8 addr; for (addr = start_addr; addr < I3C_MAX_ADDR; addr++) { - status = i3c_bus_get_addr_slot_status(bus, addr); + status = i3c_bus_get_addr_slot_status_mask(bus, addr, + I3C_ADDR_SLOT_EXT_STATUS_MASK); + if (status == I3C_ADDR_SLOT_FREE) + return addr; + } + + for (addr = start_addr; addr < I3C_MAX_ADDR; addr++) { + status = i3c_bus_get_addr_slot_status_mask(bus, addr, + I3C_ADDR_SLOT_STATUS_MASK); if (status == I3C_ADDR_SLOT_FREE) return addr; } @@ -1417,7 +1460,7 @@ static void i3c_master_put_i3c_addrs(struct i3c_dev_desc *dev) I3C_ADDR_SLOT_FREE); if (dev->boardinfo && dev->boardinfo->init_dyn_addr) - i3c_bus_set_addr_slot_status(&master->bus, dev->info.dyn_addr, + i3c_bus_set_addr_slot_status(&master->bus, dev->boardinfo->init_dyn_addr, I3C_ADDR_SLOT_FREE); } @@ -1506,16 +1549,9 @@ static int i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, u8 old_dyn_addr) { struct i3c_master_controller *master = i3c_dev_get_master(dev); - enum i3c_addr_slot_status status; int ret; - if (dev->info.dyn_addr != old_dyn_addr && - (!dev->boardinfo || - dev->info.dyn_addr != dev->boardinfo->init_dyn_addr)) { - status = i3c_bus_get_addr_slot_status(&master->bus, - dev->info.dyn_addr); - if (status != I3C_ADDR_SLOT_FREE) - return -EBUSY; + if (dev->info.dyn_addr != old_dyn_addr) { i3c_bus_set_addr_slot_status(&master->bus, dev->info.dyn_addr, I3C_ADDR_SLOT_I3C_DEV); @@ -1918,9 +1954,11 @@ static int i3c_master_bus_init(struct i3c_master_controller *master) goto err_rstdaa; } - i3c_bus_set_addr_slot_status(&master->bus, - i3cboardinfo->init_dyn_addr, - I3C_ADDR_SLOT_I3C_DEV); + /* Do not mark as occupied until real device exist in bus */ + i3c_bus_set_addr_slot_status_mask(&master->bus, + i3cboardinfo->init_dyn_addr, + I3C_ADDR_SLOT_EXT_DESIRED, + I3C_ADDR_SLOT_EXT_STATUS_MASK); /* * Only try to create/attach devices that have a static @@ -2051,11 +2089,16 @@ int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, ibireq.max_payload_len = olddev->ibi->max_payload_len; ibireq.num_slots = olddev->ibi->num_slots; - if (olddev->ibi->enabled) { + if (olddev->ibi->enabled) enable_ibi = true; - i3c_dev_disable_ibi_locked(olddev); - } - + /* + * The olddev should not receive any commands on the + * i3c bus as it does not exist and has been assigned + * a new address. This will result in NACK or timeout. + * So, update the olddev->ibi->enabled flag to false + * to avoid DISEC with OldAddr. + */ + olddev->ibi->enabled = false; i3c_dev_free_ibi_locked(olddev); } mutex_unlock(&olddev->ibi_lock); @@ -2083,7 +2126,8 @@ int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, else expected_dyn_addr = newdev->info.dyn_addr; - if (newdev->info.dyn_addr != expected_dyn_addr) { + if (newdev->info.dyn_addr != expected_dyn_addr && + i3c_bus_get_addr_slot_status(&master->bus, expected_dyn_addr) == I3C_ADDR_SLOT_FREE) { /* * Try to apply the expected dynamic address. If it fails, keep * the address assigned by the master. diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 8d694672c110..5b5c2e4bdc51 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -220,6 +220,14 @@ #define XFER_TIMEOUT (msecs_to_jiffies(1000)) #define RPM_AUTOSUSPEND_TIMEOUT 1000 /* ms */ + +/* Timing values to configure 12.5MHz frequency */ +#define AMD_I3C_OD_TIMING 0x4C007C +#define AMD_I3C_PP_TIMING 0x8001A + +/* List of quirks */ +#define AMD_I3C_OD_PP_TIMING BIT(1) + struct dw_i3c_cmd { u32 cmd_lo; u32 cmd_hi; @@ -794,6 +802,12 @@ static int dw_i3c_ccc_get(struct dw_i3c_master *master, struct i3c_ccc_cmd *ccc) return ret; } +static void amd_configure_od_pp_quirk(struct dw_i3c_master *master) +{ + master->i3c_od_timing = AMD_I3C_OD_TIMING; + master->i3c_pp_timing = AMD_I3C_PP_TIMING; +} + static int dw_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, struct i3c_ccc_cmd *ccc) { @@ -803,6 +817,13 @@ static int dw_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, if (ccc->id == I3C_CCC_ENTDAA) return -EINVAL; + /* AMD platform specific OD and PP timings */ + if (master->quirks & AMD_I3C_OD_PP_TIMING) { + amd_configure_od_pp_quirk(master); + writel(master->i3c_pp_timing, master->regs + SCL_I3C_PP_TIMING); + writel(master->i3c_od_timing, master->regs + SCL_I3C_OD_TIMING); + } + ret = pm_runtime_resume_and_get(master->dev); if (ret < 0) { dev_err(master->dev, @@ -1602,6 +1623,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, master->maxdevs = ret >> 16; master->free_pos = GENMASK(master->maxdevs - 1, 0); + master->quirks = (unsigned long)device_get_match_data(&pdev->dev); + INIT_WORK(&master->hj_work, dw_i3c_hj_work); ret = i3c_master_register(&master->base, &pdev->dev, &dw_mipi_i3c_ops, false); @@ -1675,6 +1698,10 @@ static void dw_i3c_master_restore_addrs(struct dw_i3c_master *master) static void dw_i3c_master_restore_timing_regs(struct dw_i3c_master *master) { + /* AMD platform specific OD and PP timings */ + if (master->quirks & AMD_I3C_OD_PP_TIMING) + amd_configure_od_pp_quirk(master); + writel(master->i3c_pp_timing, master->regs + SCL_I3C_PP_TIMING); writel(master->bus_free_timing, master->regs + BUS_FREE_TIMING); writel(master->i3c_od_timing, master->regs + SCL_I3C_OD_TIMING); @@ -1748,12 +1775,19 @@ static const struct of_device_id dw_i3c_master_of_match[] = { }; MODULE_DEVICE_TABLE(of, dw_i3c_master_of_match); +static const struct acpi_device_id amd_i3c_device_match[] = { + { "AMDI0015", AMD_I3C_OD_PP_TIMING }, + { } +}; +MODULE_DEVICE_TABLE(acpi, amd_i3c_device_match); + static struct platform_driver dw_i3c_driver = { .probe = dw_i3c_probe, .remove_new = dw_i3c_remove, .driver = { .name = "dw-i3c-master", .of_match_table = dw_i3c_master_of_match, + .acpi_match_table = amd_i3c_device_match, .pm = &dw_i3c_pm_ops, }, }; diff --git a/drivers/i3c/master/dw-i3c-master.h b/drivers/i3c/master/dw-i3c-master.h index 219ff815d3a7..c5cb695c16ab 100644 --- a/drivers/i3c/master/dw-i3c-master.h +++ b/drivers/i3c/master/dw-i3c-master.h @@ -50,6 +50,7 @@ struct dw_i3c_master { u32 bus_free_timing; u32 i2c_fm_timing; u32 i2c_fmp_timing; + u32 quirks; /* * Per-device hardware data, used to manage the device address table * (DAT) diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c index a82c47c9986d..e6e482a259b4 100644 --- a/drivers/i3c/master/mipi-i3c-hci/core.c +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -80,8 +80,6 @@ #define INTR_HC_CMD_SEQ_UFLOW_STAT BIT(12) /* Cmd Sequence Underflow */ #define INTR_HC_RESET_CANCEL BIT(11) /* HC Cancelled Reset */ #define INTR_HC_INTERNAL_ERR BIT(10) /* HC Internal Error */ -#define INTR_HC_PIO BIT(8) /* cascaded PIO interrupt */ -#define INTR_HC_RINGS GENMASK(7, 0) #define DAT_SECTION 0x30 /* Device Address Table */ #define DAT_ENTRY_SIZE GENMASK(31, 28) @@ -438,7 +436,8 @@ static int i3c_hci_attach_i3c_dev(struct i3c_dev_desc *dev) kfree(dev_data); return ret; } - mipi_i3c_hci_dat_v1.set_dynamic_addr(hci, ret, dev->info.dyn_addr); + mipi_i3c_hci_dat_v1.set_dynamic_addr(hci, ret, + dev->info.dyn_addr ?: dev->info.static_addr); dev_data->dat_idx = ret; } i3c_dev_set_master_data(dev, dev_data); @@ -597,9 +596,6 @@ static irqreturn_t i3c_hci_irq_handler(int irq, void *dev_id) if (val) { reg_write(INTR_STATUS, val); - } else { - /* v1.0 does not have PIO cascaded notification bits */ - val |= INTR_HC_PIO; } if (val & INTR_HC_RESET_CANCEL) { @@ -610,14 +606,9 @@ static irqreturn_t i3c_hci_irq_handler(int irq, void *dev_id) dev_err(&hci->master.dev, "Host Controller Internal Error\n"); val &= ~INTR_HC_INTERNAL_ERR; } - if (val & INTR_HC_PIO) { - hci->io->irq_handler(hci, 0); - val &= ~INTR_HC_PIO; - } - if (val & INTR_HC_RINGS) { - hci->io->irq_handler(hci, val & INTR_HC_RINGS); - val &= ~INTR_HC_RINGS; - } + + hci->io->irq_handler(hci); + if (val) dev_err(&hci->master.dev, "unexpected INTR_STATUS %#x\n", val); else diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c index a918e96b21fd..e8e56a8d2057 100644 --- a/drivers/i3c/master/mipi-i3c-hci/dma.c +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -159,10 +159,10 @@ static void hci_dma_cleanup(struct i3c_hci *hci) for (i = 0; i < rings->total; i++) { rh = &rings->headers[i]; + rh_reg_write(INTR_SIGNAL_ENABLE, 0); rh_reg_write(RING_CONTROL, 0); rh_reg_write(CR_SETUP, 0); rh_reg_write(IBI_SETUP, 0); - rh_reg_write(INTR_SIGNAL_ENABLE, 0); if (rh->xfer) dma_free_coherent(&hci->master.dev, @@ -733,20 +733,16 @@ done: rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks); } -static bool hci_dma_irq_handler(struct i3c_hci *hci, unsigned int mask) +static bool hci_dma_irq_handler(struct i3c_hci *hci) { struct hci_rings_data *rings = hci->io_data; unsigned int i; bool handled = false; - for (i = 0; mask && i < rings->total; i++) { + for (i = 0; i < rings->total; i++) { struct hci_rh_data *rh; u32 status; - if (!(mask & BIT(i))) - continue; - mask &= ~BIT(i); - rh = &rings->headers[i]; status = rh_reg_read(INTR_STATUS); DBG("rh%d status: %#x", i, status); diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h index aaa47ac47381..69ea1d10414b 100644 --- a/drivers/i3c/master/mipi-i3c-hci/hci.h +++ b/drivers/i3c/master/mipi-i3c-hci/hci.h @@ -115,7 +115,7 @@ static inline void hci_free_xfer(struct hci_xfer *xfer, unsigned int n) /* This abstracts PIO vs DMA operations */ struct hci_io_ops { - bool (*irq_handler)(struct i3c_hci *hci, unsigned int mask); + bool (*irq_handler)(struct i3c_hci *hci); int (*queue_xfer)(struct i3c_hci *hci, struct hci_xfer *xfer, int n); bool (*dequeue_xfer)(struct i3c_hci *hci, struct hci_xfer *xfer, int n); int (*request_ibi)(struct i3c_hci *hci, struct i3c_dev_desc *dev, diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c index d0272aa93599..2fc71e696911 100644 --- a/drivers/i3c/master/mipi-i3c-hci/pio.c +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -979,7 +979,7 @@ static void hci_pio_recycle_ibi_slot(struct i3c_hci *hci, i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); } -static bool hci_pio_irq_handler(struct i3c_hci *hci, unsigned int unused) +static bool hci_pio_irq_handler(struct i3c_hci *hci) { struct hci_pio_data *pio = hci->io_data; u32 status; diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index a7bfc678153e..c1ee3828e7ee 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -130,8 +130,8 @@ #define SVC_I3C_PPBAUD_MAX 15 #define SVC_I3C_QUICK_I2C_CLK 4170000 -#define SVC_I3C_EVENT_IBI BIT(0) -#define SVC_I3C_EVENT_HOTJOIN BIT(1) +#define SVC_I3C_EVENT_IBI GENMASK(7, 0) +#define SVC_I3C_EVENT_HOTJOIN BIT(31) struct svc_i3c_cmd { u8 addr; @@ -214,7 +214,7 @@ struct svc_i3c_master { spinlock_t lock; } ibi; struct mutex lock; - int enabled_events; + u32 enabled_events; u32 mctrl_config; }; @@ -388,10 +388,11 @@ static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master, return 0; } -static void svc_i3c_master_ack_ibi(struct svc_i3c_master *master, +static int svc_i3c_master_ack_ibi(struct svc_i3c_master *master, bool mandatory_byte) { unsigned int ibi_ack_nack; + u32 reg; ibi_ack_nack = SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK; if (mandatory_byte) @@ -400,13 +401,43 @@ static void svc_i3c_master_ack_ibi(struct svc_i3c_master *master, ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE; writel(ibi_ack_nack, master->regs + SVC_I3C_MCTRL); + + return readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg), 1, 1000); + } -static void svc_i3c_master_nack_ibi(struct svc_i3c_master *master) +static int svc_i3c_master_nack_ibi(struct svc_i3c_master *master) { + int ret; + u32 reg; + writel(SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK | SVC_I3C_MCTRL_IBIRESP_NACK, master->regs + SVC_I3C_MCTRL); + + ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg), 1, 1000); + return ret; +} + +static int svc_i3c_master_handle_ibi_won(struct svc_i3c_master *master, u32 mstatus) +{ + u32 ibitype; + int ret = 0; + + ibitype = SVC_I3C_MSTATUS_IBITYPE(mstatus); + + writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); + + /* Hardware can't auto emit NACK for hot join and master request */ + switch (ibitype) { + case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN: + case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: + ret = svc_i3c_master_nack_ibi(master); + } + + return ret; } static void svc_i3c_master_ibi_work(struct work_struct *work) @@ -418,7 +449,16 @@ static void svc_i3c_master_ibi_work(struct work_struct *work) u32 status, val; int ret; - mutex_lock(&master->lock); + /* + * According to I3C spec ver 1.1, 09-Jun-2021, section 5.1.2.5: + * + * The I3C Controller shall hold SCL low while the Bus is in ACK/NACK Phase of I3C/I2C + * transfer. But maximum stall time is 100us. The IRQs have to be disabled to prevent + * schedule during the whole I3C transaction, otherwise, the I3C bus timeout may happen if + * any irq or schedule happen during transaction. + */ + guard(spinlock_irqsave)(&master->xferqueue.lock); + /* * IBIWON may be set before SVC_I3C_MCTRL_REQUEST_AUTO_IBI, causing * readl_relaxed_poll_timeout() to return immediately. Consequently, @@ -438,8 +478,8 @@ static void svc_i3c_master_ibi_work(struct work_struct *work) master->regs + SVC_I3C_MCTRL); /* Wait for IBIWON, should take approximately 100us */ - ret = readl_relaxed_poll_timeout(master->regs + SVC_I3C_MSTATUS, val, - SVC_I3C_MSTATUS_IBIWON(val), 0, 1000); + ret = readl_relaxed_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, val, + SVC_I3C_MSTATUS_IBIWON(val), 0, 100); if (ret) { dev_err(master->dev, "Timeout when polling for IBIWON\n"); svc_i3c_master_emit_stop(master); @@ -511,7 +551,6 @@ static void svc_i3c_master_ibi_work(struct work_struct *work) reenable_ibis: svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART); - mutex_unlock(&master->lock); } static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id) @@ -854,6 +893,9 @@ static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master, int ret, i; while (true) { + /* clean SVC_I3C_MINT_IBIWON w1c bits */ + writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); + /* SVC_I3C_MCTRL_REQUEST_PROC_DAA have two mode, ENTER DAA or PROCESS DAA. * * ENTER DAA: @@ -905,6 +947,11 @@ static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master, ret = svc_i3c_master_readb(master, data, 2); if (ret) break; + } else if (SVC_I3C_MSTATUS_IBIWON(reg)) { + ret = svc_i3c_master_handle_ibi_won(master, reg); + if (ret) + break; + continue; } else if (SVC_I3C_MSTATUS_MCTRLDONE(reg)) { if (SVC_I3C_MSTATUS_STATE_IDLE(reg) && SVC_I3C_MSTATUS_COMPLETE(reg)) { @@ -1056,12 +1103,27 @@ static int svc_i3c_master_do_daa(struct i3c_master_controller *m) if (ret) goto rpm_out; - /* Register all devices who participated to the core */ - for (i = 0; i < dev_nb; i++) { - ret = i3c_master_add_i3c_dev_locked(m, addrs[i]); - if (ret) - goto rpm_out; - } + /* + * Register all devices who participated to the core + * + * If two devices (A and B) are detected in DAA and address 0xa is assigned to + * device A and 0xb to device B, a failure in i3c_master_add_i3c_dev_locked() + * for device A (addr: 0xa) could prevent device B (addr: 0xb) from being + * registered on the bus. The I3C stack might still consider 0xb a free + * address. If a subsequent Hotjoin occurs, 0xb might be assigned to Device A, + * causing both devices A and B to use the same address 0xb, violating the I3C + * specification. + * + * The return value for i3c_master_add_i3c_dev_locked() should not be checked + * because subsequent steps will scan the entire I3C bus, independent of + * whether i3c_master_add_i3c_dev_locked() returns success. + * + * If device A registration fails, there is still a chance to register device + * B. i3c_master_add_i3c_dev_locked() can reset DAA if a failure occurs while + * retrieving device information. + */ + for (i = 0; i < dev_nb; i++) + i3c_master_add_i3c_dev_locked(m, addrs[i]); /* Configure IBI auto-rules */ ret = svc_i3c_update_ibirules(master); @@ -1163,6 +1225,26 @@ static int svc_i3c_master_xfer(struct svc_i3c_master *master, if (ret) goto emit_stop; + /* + * According to I3C spec ver 1.1.1, 5.1.2.2.3 Consequence of Controller Starting a + * Frame with I3C Target Address. + * + * The I3C Controller normally should start a Frame, the Address may be arbitrated, + * and so the Controller shall monitor to see whether an In-Band Interrupt request, + * a Controller Role Request (i.e., Secondary Controller requests to become the + * Active Controller), or a Hot-Join Request has been made. + * + * If missed IBIWON check, the wrong data will be return. When IBIWON happen, issue + * repeat start. Address arbitrate only happen at START, never happen at REPEAT + * start. + */ + if (SVC_I3C_MSTATUS_IBIWON(reg)) { + ret = svc_i3c_master_handle_ibi_won(master, reg); + if (ret) + goto emit_stop; + continue; + } + if (readl(master->regs + SVC_I3C_MERRWARN) & SVC_I3C_MERRWARN_NACK) { /* * According to I3C Spec 1.1.1, 11-Jun-2021, section: 5.1.2.2.3. @@ -1196,24 +1278,6 @@ static int svc_i3c_master_xfer(struct svc_i3c_master *master, } } - /* - * According to I3C spec ver 1.1.1, 5.1.2.2.3 Consequence of Controller Starting a Frame - * with I3C Target Address. - * - * The I3C Controller normally should start a Frame, the Address may be arbitrated, and so - * the Controller shall monitor to see whether an In-Band Interrupt request, a Controller - * Role Request (i.e., Secondary Controller requests to become the Active Controller), or - * a Hot-Join Request has been made. - * - * If missed IBIWON check, the wrong data will be return. When IBIWON happen, return failure - * and yield the above events handler. - */ - if (SVC_I3C_MSTATUS_IBIWON(reg)) { - ret = -EAGAIN; - *actual_len = 0; - goto emit_stop; - } - if (rnw) ret = svc_i3c_master_read(master, in, xfer_len); else @@ -1624,7 +1688,7 @@ static int svc_i3c_master_enable_ibi(struct i3c_dev_desc *dev) return ret; } - master->enabled_events |= SVC_I3C_EVENT_IBI; + master->enabled_events++; svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART); return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); @@ -1636,7 +1700,7 @@ static int svc_i3c_master_disable_ibi(struct i3c_dev_desc *dev) struct svc_i3c_master *master = to_svc_i3c_master(m); int ret; - master->enabled_events &= ~SVC_I3C_EVENT_IBI; + master->enabled_events--; if (!master->enabled_events) svc_i3c_master_disable_interrupts(master); @@ -1827,8 +1891,8 @@ static int svc_i3c_master_probe(struct platform_device *pdev) rpm_disable: pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_noidle(&pdev->dev); - pm_runtime_set_suspended(&pdev->dev); pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); err_disable_clks: svc_i3c_master_unprepare_clks(master); diff --git a/drivers/net/ethernet/cavium/common/cavium_ptp.c b/drivers/net/ethernet/cavium/common/cavium_ptp.c index 9fd717b9cf69..984f0dd7b62e 100644 --- a/drivers/net/ethernet/cavium/common/cavium_ptp.c +++ b/drivers/net/ethernet/cavium/common/cavium_ptp.c @@ -239,12 +239,11 @@ static int cavium_ptp_probe(struct pci_dev *pdev, if (err) goto error_free; - err = pcim_iomap_regions(pdev, 1 << PCI_PTP_BAR_NO, pci_name(pdev)); + clock->reg_base = pcim_iomap_region(pdev, PCI_PTP_BAR_NO, pci_name(pdev)); + err = PTR_ERR_OR_ZERO(clock->reg_base); if (err) goto error_free; - clock->reg_base = pcim_iomap_table(pdev)[PCI_PTP_BAR_NO]; - spin_lock_init(&clock->spin_lock); cc = &clock->cycle_counter; @@ -292,7 +291,7 @@ error_stop: clock_cfg = readq(clock->reg_base + PTP_CLOCK_CFG); clock_cfg &= ~PTP_CLOCK_CFG_PTP_EN; writeq(clock_cfg, clock->reg_base + PTP_CLOCK_CFG); - pcim_iounmap_regions(pdev, 1 << PCI_PTP_BAR_NO); + pcim_iounmap_region(pdev, PCI_PTP_BAR_NO); error_free: devm_kfree(dev, clock); diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c index d19b3bd0866b..86f1d87a909c 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c @@ -3535,7 +3535,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev, struct iwl_trans_pcie *trans_pcie, **priv; struct iwl_trans *trans; int ret, addr_size; - void __iomem * const *table; u32 bar0; /* reassign our BAR 0 if invalid due to possible runtime PM races */ @@ -3661,22 +3660,15 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev, } } - ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME); + ret = pcim_request_all_regions(pdev, DRV_NAME); if (ret) { - dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n"); + dev_err(&pdev->dev, "Requesting all PCI BARs failed.\n"); goto out_no_pci; } - table = pcim_iomap_table(pdev); - if (!table) { - dev_err(&pdev->dev, "pcim_iomap_table failed\n"); - ret = -ENOMEM; - goto out_no_pci; - } - - trans_pcie->hw_base = table[0]; + trans_pcie->hw_base = pcim_iomap(pdev, 0, 0); if (!trans_pcie->hw_base) { - dev_err(&pdev->dev, "couldn't find IO mem in first BAR\n"); + dev_err(&pdev->dev, "Could not ioremap PCI BAR 0.\n"); ret = -ENODEV; goto out_no_pci; } diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c index 6fc9dfe82474..544d8a4d2af5 100644 --- a/drivers/ntb/hw/idt/ntb_hw_idt.c +++ b/drivers/ntb/hw/idt/ntb_hw_idt.c @@ -2671,15 +2671,20 @@ static int idt_init_pci(struct idt_ntb_dev *ndev) */ pci_set_master(pdev); - /* Request all BARs resources and map BAR0 only */ - ret = pcim_iomap_regions_request_all(pdev, 1, NTB_NAME); + /* Request all BARs resources */ + ret = pcim_request_all_regions(pdev, NTB_NAME); if (ret != 0) { dev_err(&pdev->dev, "Failed to request resources\n"); goto err_clear_master; } - /* Retrieve virtual address of BAR0 - PCI configuration space */ - ndev->cfgspc = pcim_iomap_table(pdev)[0]; + /* ioremap BAR0 - PCI configuration space */ + ndev->cfgspc = pcim_iomap(pdev, 0, 0); + if (!ndev->cfgspc) { + dev_err(&pdev->dev, "Failed to ioremap BAR 0\n"); + ret = -ENOMEM; + goto err_clear_master; + } /* Put the IDT driver data pointer to the PCI-device private pointer */ pci_set_drvdata(pdev, ndev); diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 0d94e4a967d8..2fbd379923fd 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -173,6 +173,15 @@ config PCI_PASID If unsure, say N. +config PCIE_TPH + bool "TLP Processing Hints" + help + This option adds support for PCIe TLP Processing Hints (TPH). + TPH allows endpoint devices to provide optimization hints, such as + desired caching behavior, for requests that target memory space. + These hints, called Steering Tags, can empower the system hardware + to optimize the utilization of platform resources. + config PCI_P2PDMA bool "PCI peer-to-peer transfer support" depends on ZONE_DEVICE @@ -305,6 +314,6 @@ source "drivers/pci/hotplug/Kconfig" source "drivers/pci/controller/Kconfig" source "drivers/pci/endpoint/Kconfig" source "drivers/pci/switch/Kconfig" -source "drivers/pci/pwrctl/Kconfig" +source "drivers/pci/pwrctrl/Kconfig" endif diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 374c5c06d92f..67647f1880fb 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -9,7 +9,7 @@ obj-$(CONFIG_PCI) += access.o bus.o probe.o host-bridge.o \ obj-$(CONFIG_PCI) += msi/ obj-$(CONFIG_PCI) += pcie/ -obj-$(CONFIG_PCI) += pwrctl/ +obj-$(CONFIG_PCI) += pwrctrl/ ifdef CONFIG_PCI obj-$(CONFIG_PROC_FS) += proc.o @@ -36,6 +36,7 @@ obj-$(CONFIG_VGA_ARB) += vgaarb.o obj-$(CONFIG_PCI_DOE) += doe.o obj-$(CONFIG_PCI_DYNAMIC_OF_NODES) += of_property.o obj-$(CONFIG_PCI_NPEM) += npem.o +obj-$(CONFIG_PCIE_TPH) += tph.o # Endpoint library must be initialized before its users obj-$(CONFIG_PCI_ENDPOINT) += endpoint/ diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index 55c853686051..98910bc0fcc4 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -13,11 +13,24 @@ #include <linux/ioport.h> #include <linux/of.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/proc_fs.h> #include <linux/slab.h> #include "pci.h" +/* + * The first PCI_BRIDGE_RESOURCE_NUM PCI bus resources (those that correspond + * to P2P or CardBus bridge windows) go in a table. Additional ones (for + * buses below host bridges or subtractive decode bridges) go in the list. + * Use pci_bus_for_each_resource() to iterate through all the resources. + */ + +struct pci_bus_resource { + struct list_head list; + struct resource *res; +}; + void pci_add_resource_offset(struct list_head *resources, struct resource *res, resource_size_t offset) { @@ -46,8 +59,7 @@ void pci_free_resource_list(struct list_head *resources) } EXPORT_SYMBOL(pci_free_resource_list); -void pci_bus_add_resource(struct pci_bus *bus, struct resource *res, - unsigned int flags) +void pci_bus_add_resource(struct pci_bus *bus, struct resource *res) { struct pci_bus_resource *bus_res; @@ -58,7 +70,6 @@ void pci_bus_add_resource(struct pci_bus *bus, struct resource *res, } bus_res->res = res; - bus_res->flags = flags; list_add_tail(&bus_res->list, &bus->resources); } @@ -320,6 +331,47 @@ void __weak pcibios_resource_survey_bus(struct pci_bus *bus) { } void __weak pcibios_bus_add_device(struct pci_dev *pdev) { } +/* + * Create pwrctrl devices (if required) for the PCI devices to handle the power + * state. + */ +static void pci_pwrctrl_create_devices(struct pci_dev *dev) +{ + struct device_node *np = dev_of_node(&dev->dev); + struct device *parent = &dev->dev; + struct platform_device *pdev; + + /* + * First ensure that we are starting from a PCI bridge and it has a + * corresponding devicetree node. + */ + if (np && pci_is_bridge(dev)) { + /* + * Now look for the child PCI device nodes and create pwrctrl + * devices for them. The pwrctrl device drivers will manage the + * power state of the devices. + */ + for_each_available_child_of_node_scoped(np, child) { + /* + * First check whether the pwrctrl device really + * needs to be created or not. This is decided + * based on at least one of the power supplies + * being defined in the devicetree node of the + * device. + */ + if (!of_pci_supply_present(child)) { + pci_dbg(dev, "skipping OF node: %s\n", child->name); + return; + } + + /* Now create the pwrctrl device */ + pdev = of_platform_device_create(child, NULL, parent); + if (!pdev) + pci_err(dev, "failed to create OF node: %s\n", child->name); + } + } +} + /** * pci_bus_add_device - start driver for a single device * @dev: device to add @@ -329,6 +381,7 @@ void __weak pcibios_bus_add_device(struct pci_dev *pdev) { } void pci_bus_add_device(struct pci_dev *dev) { struct device_node *dn = dev->dev.of_node; + struct platform_device *pdev; int retval; /* @@ -343,20 +396,28 @@ void pci_bus_add_device(struct pci_dev *dev) pci_proc_attach_device(dev); pci_bridge_d3_update(dev); + pci_pwrctrl_create_devices(dev); + + /* + * If the PCI device is associated with a pwrctrl device with a + * power supply, create a device link between the PCI device and + * pwrctrl device. This ensures that pwrctrl drivers are probed + * before PCI client drivers. + */ + pdev = of_find_device_by_node(dn); + if (pdev && of_pci_supply_present(dn)) { + if (!device_link_add(&dev->dev, &pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER)) + pci_err(dev, "failed to add device link to power control device %s\n", + pdev->name); + } + dev->match_driver = !dn || of_device_is_available(dn); retval = device_attach(&dev->dev); if (retval < 0 && retval != -EPROBE_DEFER) pci_warn(dev, "device attach failed (%d)\n", retval); - pci_dev_assign_added(dev, true); - - if (dev_of_node(&dev->dev) && pci_is_bridge(dev)) { - retval = of_platform_populate(dev_of_node(&dev->dev), NULL, NULL, - &dev->dev); - if (retval) - pci_err(dev, "failed to populate child OF nodes (%d)\n", - retval); - } + pci_dev_assign_added(dev); } EXPORT_SYMBOL_GPL(pci_bus_add_device); @@ -389,41 +450,23 @@ void pci_bus_add_devices(const struct pci_bus *bus) } EXPORT_SYMBOL(pci_bus_add_devices); -static void __pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), - void *userdata, bool locked) +static int __pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), + void *userdata) { struct pci_dev *dev; - struct pci_bus *bus; - struct list_head *next; - int retval; + int ret = 0; - bus = top; - if (!locked) - down_read(&pci_bus_sem); - next = top->devices.next; - for (;;) { - if (next == &bus->devices) { - /* end of this bus, go up or finish */ - if (bus == top) + list_for_each_entry(dev, &top->devices, bus_list) { + ret = cb(dev, userdata); + if (ret) + break; + if (dev->subordinate) { + ret = __pci_walk_bus(dev->subordinate, cb, userdata); + if (ret) break; - next = bus->self->bus_list.next; - bus = bus->self->bus; - continue; } - dev = list_entry(next, struct pci_dev, bus_list); - if (dev->subordinate) { - /* this is a pci-pci bridge, do its devices next */ - next = dev->subordinate->devices.next; - bus = dev->subordinate; - } else - next = dev->bus_list.next; - - retval = cb(dev, userdata); - if (retval) - break; } - if (!locked) - up_read(&pci_bus_sem); + return ret; } /** @@ -441,7 +484,9 @@ static void __pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void */ void pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), void *userdata) { - __pci_walk_bus(top, cb, userdata, false); + down_read(&pci_bus_sem); + __pci_walk_bus(top, cb, userdata); + up_read(&pci_bus_sem); } EXPORT_SYMBOL_GPL(pci_walk_bus); @@ -449,9 +494,8 @@ void pci_walk_bus_locked(struct pci_bus *top, int (*cb)(struct pci_dev *, void * { lockdep_assert_held(&pci_bus_sem); - __pci_walk_bus(top, cb, userdata, true); + __pci_walk_bus(top, cb, userdata); } -EXPORT_SYMBOL_GPL(pci_walk_bus_locked); struct pci_bus *pci_bus_get(struct pci_bus *bus) { diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 284f2e0e4d26..0341d51d6aed 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -386,6 +386,13 @@ static const struct j721e_pcie_data j784s4_pcie_ep_data = { .max_lanes = 4, }; +static const struct j721e_pcie_data j722s_pcie_rc_data = { + .mode = PCI_MODE_RC, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .byte_access_allowed = true, + .max_lanes = 1, +}; + static const struct of_device_id of_j721e_pcie_match[] = { { .compatible = "ti,j721e-pcie-host", @@ -419,6 +426,10 @@ static const struct of_device_id of_j721e_pcie_match[] = { .compatible = "ti,j784s4-pcie-ep", .data = &j784s4_pcie_ep_data, }, + { + .compatible = "ti,j722s-pcie-host", + .data = &j722s_pcie_rc_data, + }, {}, }; @@ -572,15 +583,14 @@ static int j721e_pcie_probe(struct platform_device *pdev) pcie->refclk = clk; /* - * The "Power Sequencing and Reset Signal Timings" table of the - * PCI Express Card Electromechanical Specification, Revision - * 5.1, Section 2.9.2, Symbol "T_PERST-CLK", indicates PERST# - * should be deasserted after minimum of 100us once REFCLK is - * stable. The REFCLK to the connector in RC mode is selected - * while enabling the PHY. So deassert PERST# after 100 us. + * Section 2.2 of the PCI Express Card Electromechanical + * Specification (Revision 5.1) mandates that the deassertion + * of the PERST# signal should be delayed by 100 ms (TPVPERL). + * This shall ensure that the power and the reference clock + * are stable. */ if (gpiod) { - fsleep(PCIE_T_PERST_CLK_US); + msleep(PCIE_T_PVPERL_MS); gpiod_set_value_cansleep(gpiod, 1); } @@ -671,15 +681,14 @@ static int j721e_pcie_resume_noirq(struct device *dev) return ret; /* - * The "Power Sequencing and Reset Signal Timings" table of the - * PCI Express Card Electromechanical Specification, Revision - * 5.1, Section 2.9.2, Symbol "T_PERST-CLK", indicates PERST# - * should be deasserted after minimum of 100us once REFCLK is - * stable. The REFCLK to the connector in RC mode is selected - * while enabling the PHY. So deassert PERST# after 100 us. + * Section 2.2 of the PCI Express Card Electromechanical + * Specification (Revision 5.1) mandates that the deassertion + * of the PERST# signal should be delayed by 100 ms (TPVPERL). + * This shall ensure that the power and the reference clock + * are stable. */ if (pcie->reset_gpio) { - fsleep(PCIE_T_PERST_CLK_US); + msleep(PCIE_T_PVPERL_MS); gpiod_set_value_cansleep(pcie->reset_gpio, 1); } @@ -712,7 +721,7 @@ static DEFINE_NOIRQ_DEV_PM_OPS(j721e_pcie_pm_ops, static struct platform_driver j721e_pcie_driver = { .probe = j721e_pcie_probe, - .remove_new = j721e_pcie_remove, + .remove = j721e_pcie_remove, .driver = { .name = "j721e-pcie", .of_match_table = of_j721e_pcie_match, diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index 4251fac5e310..204e045aed8c 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -197,7 +197,7 @@ int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie) phy_count = of_property_count_strings(np, "phy-names"); if (phy_count < 1) { - dev_err(dev, "no phy-names. PHY will not be initialized\n"); + dev_info(dev, "no \"phy-names\" property found; PHY will not be initialized\n"); pcie->phy_count = 0; return 0; } @@ -260,7 +260,7 @@ static int cdns_pcie_resume_noirq(struct device *dev) ret = cdns_pcie_enable_phy(pcie); if (ret) { - dev_err(dev, "failed to enable phy\n"); + dev_err(dev, "failed to enable PHY\n"); return ret; } diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c index fa45da28a218..6a830166d37f 100644 --- a/drivers/pci/controller/dwc/pci-exynos.c +++ b/drivers/pci/controller/dwc/pci-exynos.c @@ -383,7 +383,7 @@ static const struct of_device_id exynos_pcie_of_match[] = { static struct platform_driver exynos_pcie_driver = { .probe = exynos_pcie_probe, - .remove_new = exynos_pcie_remove, + .remove = exynos_pcie_remove, .driver = { .name = "exynos-pcie", .of_match_table = exynos_pcie_of_match, diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 808d1f105417..c8d5c90aa4d4 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -82,6 +82,11 @@ enum imx_pcie_variants { #define IMX_PCIE_FLAG_HAS_SERDES BIT(6) #define IMX_PCIE_FLAG_SUPPORT_64BIT BIT(7) #define IMX_PCIE_FLAG_CPU_ADDR_FIXUP BIT(8) +/* + * Because of ERR005723 (PCIe does not support L2 power down) we need to + * workaround suspend resume on some devices which are affected by this errata. + */ +#define IMX_PCIE_FLAG_BROKEN_SUSPEND BIT(9) #define imx_check_flag(pci, val) (pci->drvdata->flags & val) @@ -1237,9 +1242,19 @@ static int imx_pcie_suspend_noirq(struct device *dev) return 0; imx_pcie_msi_save_restore(imx_pcie, true); - imx_pcie_pm_turnoff(imx_pcie); - imx_pcie_stop_link(imx_pcie->pci); - imx_pcie_host_exit(pp); + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) { + /* + * The minimum for a workaround would be to set PERST# and to + * set the PCIE_TEST_PD flag. However, we can also disable the + * clock which saves some power. + */ + imx_pcie_assert_core_reset(imx_pcie); + imx_pcie->drvdata->enable_ref_clk(imx_pcie, false); + } else { + imx_pcie_pm_turnoff(imx_pcie); + imx_pcie_stop_link(imx_pcie->pci); + imx_pcie_host_exit(pp); + } return 0; } @@ -1253,14 +1268,32 @@ static int imx_pcie_resume_noirq(struct device *dev) if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND)) return 0; - ret = imx_pcie_host_init(pp); - if (ret) - return ret; - imx_pcie_msi_save_restore(imx_pcie, false); - dw_pcie_setup_rc(pp); + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) { + ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true); + if (ret) + return ret; + ret = imx_pcie_deassert_core_reset(imx_pcie); + if (ret) + return ret; + /* + * Using PCIE_TEST_PD seems to disable MSI and powers down the + * root complex. This is why we have to setup the rc again and + * why we have to restore the MSI register. + */ + ret = dw_pcie_setup_rc(&imx_pcie->pci->pp); + if (ret) + return ret; + imx_pcie_msi_save_restore(imx_pcie, false); + } else { + ret = imx_pcie_host_init(pp); + if (ret) + return ret; + imx_pcie_msi_save_restore(imx_pcie, false); + dw_pcie_setup_rc(pp); - if (imx_pcie->link_is_up) - imx_pcie_start_link(imx_pcie->pci); + if (imx_pcie->link_is_up) + imx_pcie_start_link(imx_pcie->pci); + } return 0; } @@ -1485,7 +1518,9 @@ static const struct imx_pcie_drvdata drvdata[] = { [IMX6Q] = { .variant = IMX6Q, .flags = IMX_PCIE_FLAG_IMX_PHY | - IMX_PCIE_FLAG_IMX_SPEED_CHANGE, + IMX_PCIE_FLAG_IMX_SPEED_CHANGE | + IMX_PCIE_FLAG_BROKEN_SUSPEND | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .dbi_length = 0x200, .gpr = "fsl,imx6q-iomuxc-gpr", .clk_names = imx6q_clks, diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index 2219b1a866fa..63bd5003da45 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -455,6 +455,17 @@ static void __iomem *ks_pcie_other_map_bus(struct pci_bus *bus, struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); u32 reg; + /* + * Checking whether the link is up here is a last line of defense + * against platforms that forward errors on the system bus as + * SError upon PCI configuration transactions issued when the link + * is down. This check is racy by definition and does not stop + * the system from triggering an SError if the link goes down + * after this check is performed. + */ + if (!dw_pcie_link_up(pci)) + return NULL; + reg = CFG_BUS(bus->number) | CFG_DEVICE(PCI_SLOT(devfn)) | CFG_FUNC(PCI_FUNC(devfn)); if (!pci_is_root_bus(bus->parent)) @@ -1093,6 +1104,7 @@ static int ks_pcie_am654_set_mode(struct device *dev, static const struct ks_pcie_of_data ks_pcie_rc_of_data = { .host_ops = &ks_pcie_host_ops, + .mode = DW_PCIE_RC_TYPE, .version = DW_PCIE_VER_365A, }; @@ -1363,7 +1375,7 @@ static void ks_pcie_remove(struct platform_device *pdev) static struct platform_driver ks_pcie_driver = { .probe = ks_pcie_probe, - .remove_new = ks_pcie_remove, + .remove = ks_pcie_remove, .driver = { .name = "keystone-pcie", .of_match_table = ks_pcie_of_match, diff --git a/drivers/pci/controller/dwc/pcie-bt1.c b/drivers/pci/controller/dwc/pcie-bt1.c index 76d0ddea8007..1340edc18d12 100644 --- a/drivers/pci/controller/dwc/pcie-bt1.c +++ b/drivers/pci/controller/dwc/pcie-bt1.c @@ -632,7 +632,7 @@ MODULE_DEVICE_TABLE(of, bt1_pcie_of_match); static struct platform_driver bt1_pcie_driver = { .probe = bt1_pcie_probe, - .remove_new = bt1_pcie_remove, + .remove = bt1_pcie_remove, .driver = { .name = "bt1-pcie", .of_match_table = bt1_pcie_of_match, diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 43ba5c6738df..f3ac7d46a855 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -268,6 +268,20 @@ static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, return -EINVAL; } +static u64 dw_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, + size_t *pci_size, size_t *offset) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u64 mask = pci->region_align - 1; + size_t ofst = pci_addr & mask; + + *pci_size = ALIGN(ofst + *pci_size, epc->mem->window.page_size); + *offset = ofst; + + return pci_addr & ~mask; +} + static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, phys_addr_t addr) { @@ -280,6 +294,7 @@ static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, if (ret < 0) return; + ep->outbound_addr[atu_index] = 0; dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, atu_index); clear_bit(atu_index, ep->ob_window_map); } @@ -444,6 +459,7 @@ static const struct pci_epc_ops epc_ops = { .write_header = dw_pcie_ep_write_header, .set_bar = dw_pcie_ep_set_bar, .clear_bar = dw_pcie_ep_clear_bar, + .align_addr = dw_pcie_ep_align_addr, .map_addr = dw_pcie_ep_map_addr, .unmap_addr = dw_pcie_ep_unmap_addr, .set_msi = dw_pcie_ep_set_msi, @@ -488,7 +504,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u32 msg_addr_lower, msg_addr_upper, reg; struct dw_pcie_ep_func *ep_func; struct pci_epc *epc = ep->epc; - unsigned int aligned_offset; + size_t map_size = sizeof(u32); + size_t offset; u16 msg_ctrl, msg_data; bool has_upper; u64 msg_addr; @@ -516,14 +533,13 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, } msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; - aligned_offset = msg_addr & (epc->mem->window.page_size - 1); - msg_addr = ALIGN_DOWN(msg_addr, epc->mem->window.page_size); + msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, - epc->mem->window.page_size); + map_size); if (ret) return ret; - writel(msg_data | (interrupt_num - 1), ep->msi_mem + aligned_offset); + writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); @@ -574,8 +590,9 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, struct pci_epf_msix_tbl *msix_tbl; struct dw_pcie_ep_func *ep_func; struct pci_epc *epc = ep->epc; + size_t map_size = sizeof(u32); + size_t offset; u32 reg, msg_data, vec_ctrl; - unsigned int aligned_offset; u32 tbl_offset; u64 msg_addr; int ret; @@ -600,14 +617,13 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, return -EPERM; } - aligned_offset = msg_addr & (epc->mem->window.page_size - 1); - msg_addr = ALIGN_DOWN(msg_addr, epc->mem->window.page_size); + msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, - epc->mem->window.page_size); + map_size); if (ret) return ret; - writel(msg_data, ep->msi_mem + aligned_offset); + writel(msg_data, ep->msi_mem + offset); dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); @@ -689,7 +705,7 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) * for 1 MB BAR size only. */ for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) - dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, 0x0); + dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, BIT(4)); } dw_pcie_setup(pci); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 3e41865c7290..d2291c3ceb8b 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -474,8 +474,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp) if (pci_msi_enabled()) { pp->has_msi_ctrl = !(pp->ops->msi_init || - of_property_read_bool(np, "msi-parent") || - of_property_read_bool(np, "msi-map")); + of_property_present(np, "msi-parent") || + of_property_present(np, "msi-map")); /* * For the has_msi_ctrl case the default assignment is handled diff --git a/drivers/pci/controller/dwc/pcie-histb.c b/drivers/pci/controller/dwc/pcie-histb.c index 7a11c618b9d9..615a0e3e6d7e 100644 --- a/drivers/pci/controller/dwc/pcie-histb.c +++ b/drivers/pci/controller/dwc/pcie-histb.c @@ -439,7 +439,7 @@ MODULE_DEVICE_TABLE(of, histb_pcie_of_match); static struct platform_driver histb_pcie_platform_driver = { .probe = histb_pcie_probe, - .remove_new = histb_pcie_remove, + .remove = histb_pcie_remove, .driver = { .name = "histb-pcie", .of_match_table = histb_pcie_of_match, diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c index 676d2aba4fbd..9b53b8f6f268 100644 --- a/drivers/pci/controller/dwc/pcie-intel-gw.c +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -443,7 +443,7 @@ static const struct of_device_id of_intel_pcie_match[] = { static struct platform_driver intel_pcie_driver = { .probe = intel_pcie_probe, - .remove_new = intel_pcie_remove, + .remove = intel_pcie_remove, .driver = { .name = "intel-gw-pcie", .of_match_table = of_intel_pcie_match, diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c index 85a2c77b1835..1b2088acb538 100644 --- a/drivers/pci/controller/dwc/pcie-kirin.c +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -769,7 +769,7 @@ static int kirin_pcie_probe(struct platform_device *pdev) static struct platform_driver kirin_pcie_driver = { .probe = kirin_pcie_probe, - .remove_new = kirin_pcie_remove, + .remove = kirin_pcie_remove, .driver = { .name = "kirin-pcie", .of_match_table = kirin_pcie_match, diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index e588fcc54589..c08f64d7a825 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -396,6 +396,10 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) return ret; } + /* Perform cleanup that requires refclk */ + pci_epc_deinit_notify(pci->ep.epc); + dw_pcie_ep_cleanup(&pci->ep); + /* Assert WAKE# to RC to indicate device is ready */ gpiod_set_value_cansleep(pcie_ep->wake, 1); usleep_range(WAKE_DELAY_US, WAKE_DELAY_US + 500); @@ -540,8 +544,6 @@ static void qcom_pcie_perst_assert(struct dw_pcie *pci) { struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); - pci_epc_deinit_notify(pci->ep.epc); - dw_pcie_ep_cleanup(&pci->ep); qcom_pcie_disable_resources(pcie_ep); pcie_ep->link_status = QCOM_PCIE_EP_LINK_DISABLED; } @@ -937,7 +939,7 @@ MODULE_DEVICE_TABLE(of, qcom_pcie_ep_match); static struct platform_driver qcom_pcie_ep_driver = { .probe = qcom_pcie_ep_probe, - .remove_new = qcom_pcie_ep_remove, + .remove = qcom_pcie_ep_remove, .driver = { .name = "qcom-pcie-ep", .of_match_table = qcom_pcie_ep_match, diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index ef44a82be058..dc102d8bd58c 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -133,6 +133,7 @@ /* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ #define PARF_INT_ALL_LINK_UP BIT(13) +#define PARF_INT_MSI_DEV_0_7 GENMASK(30, 23) /* PARF_NO_SNOOP_OVERIDE register fields */ #define WR_NO_SNOOP_OVERIDE_EN BIT(1) @@ -1364,6 +1365,16 @@ static const struct qcom_pcie_ops ops_1_9_0 = { .config_sid = qcom_pcie_config_sid_1_9_0, }; +/* Qcom IP rev.: 1.21.0 Synopsys IP rev.: 5.60a */ +static const struct qcom_pcie_ops ops_1_21_0 = { + .get_resources = qcom_pcie_get_resources_2_7_0, + .init = qcom_pcie_init_2_7_0, + .post_init = qcom_pcie_post_init_2_7_0, + .host_post_init = qcom_pcie_host_post_init_2_7_0, + .deinit = qcom_pcie_deinit_2_7_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + /* Qcom IP rev.: 2.9.0 Synopsys IP rev.: 5.00a */ static const struct qcom_pcie_ops ops_2_9_0 = { .get_resources = qcom_pcie_get_resources_2_9_0, @@ -1411,7 +1422,7 @@ static const struct qcom_pcie_cfg cfg_2_9_0 = { }; static const struct qcom_pcie_cfg cfg_sc8280xp = { - .ops = &ops_1_9_0, + .ops = &ops_1_21_0, .no_l0s = true, }; @@ -1716,7 +1727,8 @@ static int qcom_pcie_probe(struct platform_device *pdev) goto err_host_deinit; } - writel_relaxed(PARF_INT_ALL_LINK_UP, pcie->parf + PARF_INT_ALL_MASK); + writel_relaxed(PARF_INT_ALL_LINK_UP | PARF_INT_MSI_DEV_0_7, + pcie->parf + PARF_INT_ALL_MASK); } qcom_pcie_icc_opp_update(pcie); @@ -1828,6 +1840,7 @@ static const struct of_device_id qcom_pcie_match[] = { { .compatible = "qcom,pcie-ipq8064-v2", .data = &cfg_2_1_0 }, { .compatible = "qcom,pcie-ipq8074", .data = &cfg_2_3_3 }, { .compatible = "qcom,pcie-ipq8074-gen3", .data = &cfg_2_9_0 }, + { .compatible = "qcom,pcie-ipq9574", .data = &cfg_2_9_0 }, { .compatible = "qcom,pcie-msm8996", .data = &cfg_2_3_2 }, { .compatible = "qcom,pcie-qcs404", .data = &cfg_2_4_0 }, { .compatible = "qcom,pcie-sa8540p", .data = &cfg_sc8280xp }, @@ -1843,7 +1856,7 @@ static const struct of_device_id qcom_pcie_match[] = { { .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 }, { .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 }, { .compatible = "qcom,pcie-sm8550", .data = &cfg_1_9_0 }, - { .compatible = "qcom,pcie-x1e80100", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-x1e80100", .data = &cfg_sc8280xp }, { } }; diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index 3a5511c3f7d9..fc872dd35029 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -775,7 +775,7 @@ static struct platform_driver rcar_gen4_pcie_driver = { .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, .probe = rcar_gen4_pcie_probe, - .remove_new = rcar_gen4_pcie_remove, + .remove = rcar_gen4_pcie_remove, }; module_platform_driver(rcar_gen4_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index c1394f2ab63f..5103995cd6c7 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1704,9 +1704,6 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) if (ret) dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); - pci_epc_deinit_notify(pcie->pci.ep.epc); - dw_pcie_ep_cleanup(&pcie->pci.ep); - reset_control_assert(pcie->core_rst); tegra_pcie_disable_phy(pcie); @@ -1785,6 +1782,10 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) goto fail_phy; } + /* Perform cleanup that requires refclk */ + pci_epc_deinit_notify(pcie->pci.ep.epc); + dw_pcie_ep_cleanup(&pcie->pci.ep); + /* Clear any stale interrupt statuses */ appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); @@ -2493,7 +2494,7 @@ static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { static struct platform_driver tegra_pcie_dw_driver = { .probe = tegra_pcie_dw_probe, - .remove_new = tegra_pcie_dw_remove, + .remove = tegra_pcie_dw_remove, .shutdown = tegra_pcie_dw_shutdown, .driver = { .name = "tegra194-pcie", diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index a598a98247ce..a29796cce420 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -1996,7 +1996,7 @@ static struct platform_driver advk_pcie_driver = { .of_match_table = advk_pcie_of_match_table, }, .probe = advk_pcie_probe, - .remove_new = advk_pcie_remove, + .remove = advk_pcie_remove, }; module_platform_driver(advk_pcie_driver); diff --git a/drivers/pci/controller/pci-host-generic.c b/drivers/pci/controller/pci-host-generic.c index 5f06f94db7b1..4051b9b61dac 100644 --- a/drivers/pci/controller/pci-host-generic.c +++ b/drivers/pci/controller/pci-host-generic.c @@ -82,7 +82,7 @@ static struct platform_driver gen_pci_driver = { .of_match_table = gen_pci_of_match, }, .probe = pci_host_common_probe, - .remove_new = pci_host_common_remove, + .remove = pci_host_common_remove, }; module_platform_driver(gen_pci_driver); diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c index 29fe09c99e7d..46d3afe1d308 100644 --- a/drivers/pci/controller/pci-mvebu.c +++ b/drivers/pci/controller/pci-mvebu.c @@ -1727,7 +1727,7 @@ static struct platform_driver mvebu_pcie_driver = { .pm = &mvebu_pcie_pm_ops, }, .probe = mvebu_pcie_probe, - .remove_new = mvebu_pcie_remove, + .remove = mvebu_pcie_remove, }; module_platform_driver(mvebu_pcie_driver); diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index d7517c3976e7..b3cdbc5927de 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -1460,7 +1460,7 @@ static int tegra_pcie_get_resources(struct tegra_pcie *pcie) pcie->cs = *res; /* constrain configuration space to 4 KiB */ - pcie->cs.end = pcie->cs.start + SZ_4K - 1; + resource_set_size(&pcie->cs, SZ_4K); pcie->cfg = devm_ioremap_resource(dev, &pcie->cs); if (IS_ERR(pcie->cfg)) { @@ -2800,6 +2800,6 @@ static struct platform_driver tegra_pcie_driver = { .pm = &tegra_pcie_pm_ops, }, .probe = tegra_pcie_probe, - .remove_new = tegra_pcie_remove, + .remove = tegra_pcie_remove, }; module_platform_driver(tegra_pcie_driver); diff --git a/drivers/pci/controller/pci-thunder-pem.c b/drivers/pci/controller/pci-thunder-pem.c index 06a9855cb431..f1bd5de67997 100644 --- a/drivers/pci/controller/pci-thunder-pem.c +++ b/drivers/pci/controller/pci-thunder-pem.c @@ -400,9 +400,9 @@ static int thunder_pem_acpi_init(struct pci_config_window *cfg) * Reserve 64K size PEM specific resources. The full 16M range * size is required for thunder_pem_init() call. */ - res_pem->end = res_pem->start + SZ_64K - 1; + resource_set_size(res_pem, SZ_64K); thunder_pem_reserve_range(dev, root->segment, res_pem); - res_pem->end = res_pem->start + SZ_16M - 1; + resource_set_size(res_pem, SZ_16M); /* Reserve PCI configuration space as well. */ thunder_pem_reserve_range(dev, root->segment, &cfg->res); diff --git a/drivers/pci/controller/pci-xgene-msi.c b/drivers/pci/controller/pci-xgene-msi.c index 3ce38dfd0d29..88c0977bc41a 100644 --- a/drivers/pci/controller/pci-xgene-msi.c +++ b/drivers/pci/controller/pci-xgene-msi.c @@ -518,7 +518,7 @@ static struct platform_driver xgene_msi_driver = { .of_match_table = xgene_msi_match_table, }, .probe = xgene_msi_probe, - .remove_new = xgene_msi_remove, + .remove = xgene_msi_remove, }; static int __init xgene_pcie_msi_init(void) diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c index e36a6e158d23..e1cee3c0575f 100644 --- a/drivers/pci/controller/pcie-altera-msi.c +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -267,7 +267,7 @@ static struct platform_driver altera_msi_driver = { .of_match_table = altera_msi_of_match, }, .probe = altera_msi_probe, - .remove_new = altera_msi_remove, + .remove = altera_msi_remove, }; static int __init altera_msi_init(void) diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c index 650b2dd81c48..eb55a7f8573a 100644 --- a/drivers/pci/controller/pcie-altera.c +++ b/drivers/pci/controller/pcie-altera.c @@ -815,10 +815,10 @@ static void altera_pcie_remove(struct platform_device *pdev) } static struct platform_driver altera_pcie_driver = { - .probe = altera_pcie_probe, - .remove_new = altera_pcie_remove, + .probe = altera_pcie_probe, + .remove = altera_pcie_remove, .driver = { - .name = "altera-pcie", + .name = "altera-pcie", .of_match_table = altera_pcie_of_match, }, }; diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 9321280f6edb..e733a27dc8df 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -1928,7 +1928,7 @@ static const struct dev_pm_ops brcm_pcie_pm_ops = { static struct platform_driver brcm_pcie_driver = { .probe = brcm_pcie_probe, - .remove_new = brcm_pcie_remove, + .remove = brcm_pcie_remove, .driver = { .name = "brcm-pcie", .of_match_table = brcm_pcie_match, diff --git a/drivers/pci/controller/pcie-hisi-error.c b/drivers/pci/controller/pcie-hisi-error.c index ad9d5ffcd9e3..aaf1ed2b6e59 100644 --- a/drivers/pci/controller/pcie-hisi-error.c +++ b/drivers/pci/controller/pcie-hisi-error.c @@ -317,7 +317,7 @@ static struct platform_driver hisi_pcie_error_handler_driver = { .acpi_match_table = hisi_pcie_acpi_match, }, .probe = hisi_pcie_error_handler_probe, - .remove_new = hisi_pcie_error_handler_remove, + .remove = hisi_pcie_error_handler_remove, }; module_platform_driver(hisi_pcie_error_handler_driver); diff --git a/drivers/pci/controller/pcie-iproc-platform.c b/drivers/pci/controller/pcie-iproc-platform.c index 4e6aa882a567..0cb78c583c7e 100644 --- a/drivers/pci/controller/pcie-iproc-platform.c +++ b/drivers/pci/controller/pcie-iproc-platform.c @@ -134,7 +134,7 @@ static struct platform_driver iproc_pltfm_pcie_driver = { .of_match_table = of_match_ptr(iproc_pcie_of_match_table), }, .probe = iproc_pltfm_pcie_probe, - .remove_new = iproc_pltfm_pcie_remove, + .remove = iproc_pltfm_pcie_remove, .shutdown = iproc_pltfm_pcie_shutdown, }; module_platform_driver(iproc_pltfm_pcie_driver); diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index 66ce4b5d309b..be52e3a123ab 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -28,7 +28,12 @@ #include "../pci.h" +#define PCIE_BASE_CFG_REG 0x14 +#define PCIE_BASE_CFG_SPEED GENMASK(15, 8) + #define PCIE_SETTING_REG 0x80 +#define PCIE_SETTING_LINK_WIDTH GENMASK(11, 8) +#define PCIE_SETTING_GEN_SUPPORT GENMASK(14, 12) #define PCIE_PCI_IDS_1 0x9c #define PCI_CLASS(class) (class << 8) #define PCIE_RC_MODE BIT(0) @@ -125,6 +130,9 @@ struct mtk_gen3_pcie; +#define PCIE_CONF_LINK2_CTL_STS (PCIE_CFG_OFFSET_ADDR + 0xb0) +#define PCIE_CONF_LINK2_LCR2_LINK_SPEED GENMASK(3, 0) + /** * struct mtk_gen3_pcie_pdata - differentiate between host generations * @power_up: pcie power_up callback @@ -160,6 +168,8 @@ struct mtk_msi_set { * @phy: PHY controller block * @clks: PCIe clocks * @num_clks: PCIe clocks count for this port + * @max_link_speed: Maximum link speed (PCIe Gen) for this port + * @num_lanes: Number of PCIe lanes for this port * @irq: PCIe controller interrupt number * @saved_irq_state: IRQ enable state saved at suspend time * @irq_lock: lock protecting IRQ register access @@ -180,6 +190,8 @@ struct mtk_gen3_pcie { struct phy *phy; struct clk_bulk_data *clks; int num_clks; + u8 max_link_speed; + u8 num_lanes; int irq; u32 saved_irq_state; @@ -381,11 +393,35 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie) int err; u32 val; - /* Set as RC mode */ + /* Set as RC mode and set controller PCIe Gen speed restriction, if any */ val = readl_relaxed(pcie->base + PCIE_SETTING_REG); val |= PCIE_RC_MODE; + if (pcie->max_link_speed) { + val &= ~PCIE_SETTING_GEN_SUPPORT; + + /* Can enable link speed support only from Gen2 onwards */ + if (pcie->max_link_speed >= 2) + val |= FIELD_PREP(PCIE_SETTING_GEN_SUPPORT, + GENMASK(pcie->max_link_speed - 2, 0)); + } + if (pcie->num_lanes) { + val &= ~PCIE_SETTING_LINK_WIDTH; + + /* Zero means one lane, each bit activates x2/x4/x8/x16 */ + if (pcie->num_lanes > 1) + val |= FIELD_PREP(PCIE_SETTING_LINK_WIDTH, + GENMASK(fls(pcie->num_lanes >> 2), 0)); + } writel_relaxed(val, pcie->base + PCIE_SETTING_REG); + /* Set Link Control 2 (LNKCTL2) speed restriction, if any */ + if (pcie->max_link_speed) { + val = readl_relaxed(pcie->base + PCIE_CONF_LINK2_CTL_STS); + val &= ~PCIE_CONF_LINK2_LCR2_LINK_SPEED; + val |= FIELD_PREP(PCIE_CONF_LINK2_LCR2_LINK_SPEED, pcie->max_link_speed); + writel_relaxed(val, pcie->base + PCIE_CONF_LINK2_CTL_STS); + } + /* Set class code */ val = readl_relaxed(pcie->base + PCIE_PCI_IDS_1); val &= ~GENMASK(31, 8); @@ -813,6 +849,7 @@ static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) struct device *dev = pcie->dev; struct platform_device *pdev = to_platform_device(dev); struct resource *regs; + u32 num_lanes; regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac"); if (!regs) @@ -858,6 +895,14 @@ static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) return pcie->num_clks; } + ret = of_property_read_u32(dev->of_node, "num-lanes", &num_lanes); + if (ret == 0) { + if (num_lanes == 0 || num_lanes > 16 || (num_lanes != 1 && num_lanes % 2)) + dev_warn(dev, "invalid num-lanes, using controller defaults\n"); + else + pcie->num_lanes = num_lanes; + } + return 0; } @@ -1004,9 +1049,21 @@ static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie) reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, pcie->phy_resets); } +static int mtk_pcie_get_controller_max_link_speed(struct mtk_gen3_pcie *pcie) +{ + u32 val; + int ret; + + val = readl_relaxed(pcie->base + PCIE_BASE_CFG_REG); + val = FIELD_GET(PCIE_BASE_CFG_SPEED, val); + ret = fls(val); + + return ret > 0 ? ret : -EINVAL; +} + static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) { - int err; + int err, max_speed; err = mtk_pcie_parse_port(pcie); if (err) @@ -1031,6 +1088,20 @@ static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) if (err) return err; + err = of_pci_get_max_link_speed(pcie->dev->of_node); + if (err) { + /* Get the maximum speed supported by the controller */ + max_speed = mtk_pcie_get_controller_max_link_speed(pcie); + + /* Set max_link_speed only if the controller supports it */ + if (max_speed >= 0 && max_speed <= err) { + pcie->max_link_speed = err; + dev_info(pcie->dev, + "maximum controller link speed Gen%d, overriding to Gen%u", + max_speed, pcie->max_link_speed); + } + } + /* Try link up */ err = mtk_pcie_startup_port(pcie); if (err) @@ -1225,7 +1296,7 @@ MODULE_DEVICE_TABLE(of, mtk_pcie_of_match); static struct platform_driver mtk_pcie_driver = { .probe = mtk_pcie_probe, - .remove_new = mtk_pcie_remove, + .remove = mtk_pcie_remove, .driver = { .name = "mtk-pcie-gen3", .of_match_table = mtk_pcie_of_match, diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 7f7d04c2ea57..3bcfc4e58ba2 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -1235,7 +1235,7 @@ MODULE_DEVICE_TABLE(of, mtk_pcie_ids); static struct platform_driver mtk_pcie_driver = { .probe = mtk_pcie_probe, - .remove_new = mtk_pcie_remove, + .remove = mtk_pcie_remove, .driver = { .name = "mtk-pcie", .of_match_table = mtk_pcie_ids, diff --git a/drivers/pci/controller/pcie-mt7621.c b/drivers/pci/controller/pcie-mt7621.c index 9b4754a45515..776caa0b1011 100644 --- a/drivers/pci/controller/pcie-mt7621.c +++ b/drivers/pci/controller/pcie-mt7621.c @@ -541,7 +541,7 @@ MODULE_DEVICE_TABLE(of, mt7621_pcie_ids); static struct platform_driver mt7621_pcie_driver = { .probe = mt7621_pcie_probe, - .remove_new = mt7621_pcie_remove, + .remove = mt7621_pcie_remove, .driver = { .name = "mt7621-pci", .of_match_table = mt7621_pcie_ids, diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index 3dd653f3d784..7c92eada04af 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -796,8 +796,8 @@ static int rcar_pcie_enable_msi(struct rcar_pcie_host *host) rcar_pci_write_reg(pcie, 0, PCIEMSIIER); /* - * Setup MSI data target using RC base address address, which - * is guaranteed to be in the low 32bit range on any R-Car HW. + * Setup MSI data target using RC base address, which is guaranteed + * to be in the low 32bit range on any R-Car HW. */ rcar_pci_write_reg(pcie, lower_32_bits(res.start) | MSIFE, PCIEMSIALR); rcar_pci_write_reg(pcie, upper_32_bits(res.start), PCIEMSIAUR); diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c index 136274533656..1064b7b06cef 100644 --- a/drivers/pci/controller/pcie-rockchip-ep.c +++ b/drivers/pci/controller/pcie-rockchip-ep.c @@ -10,12 +10,16 @@ #include <linux/configfs.h> #include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/iopoll.h> #include <linux/kernel.h> +#include <linux/irq.h> #include <linux/of.h> #include <linux/pci-epc.h> #include <linux/platform_device.h> #include <linux/pci-epf.h> #include <linux/sizes.h> +#include <linux/workqueue.h> #include "pcie-rockchip.h" @@ -48,6 +52,10 @@ struct rockchip_pcie_ep { u64 irq_pci_addr; u8 irq_pci_fn; u8 irq_pending; + int perst_irq; + bool perst_asserted; + bool link_up; + struct delayed_work link_training; }; static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, @@ -63,15 +71,25 @@ static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region)); } +static int rockchip_pcie_ep_ob_atu_num_bits(struct rockchip_pcie *rockchip, + u64 pci_addr, size_t size) +{ + int num_pass_bits = fls64(pci_addr ^ (pci_addr + size - 1)); + + return clamp(num_pass_bits, + ROCKCHIP_PCIE_AT_MIN_NUM_BITS, + ROCKCHIP_PCIE_AT_MAX_NUM_BITS); +} + static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, u32 r, u64 cpu_addr, u64 pci_addr, size_t size) { - int num_pass_bits = fls64(size - 1); + int num_pass_bits; u32 addr0, addr1, desc0; - if (num_pass_bits < 8) - num_pass_bits = 8; + num_pass_bits = rockchip_pcie_ep_ob_atu_num_bits(rockchip, + pci_addr, size); addr0 = ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) | (lower_32_bits(pci_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR); @@ -228,6 +246,28 @@ static inline u32 rockchip_ob_region(phys_addr_t addr) return (addr >> ilog2(SZ_1M)) & 0x1f; } +static u64 rockchip_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, + size_t *pci_size, size_t *addr_offset) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + size_t size = *pci_size; + u64 offset, mask; + int num_bits; + + num_bits = rockchip_pcie_ep_ob_atu_num_bits(&ep->rockchip, + pci_addr, size); + mask = (1ULL << num_bits) - 1; + + offset = pci_addr & mask; + if (size + offset > SZ_1M) + size = SZ_1M - offset; + + *pci_size = ALIGN(offset + size, ROCKCHIP_PCIE_AT_SIZE_ALIGN); + *addr_offset = offset; + + return pci_addr & ~mask; +} + static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr, u64 pci_addr, size_t size) @@ -236,6 +276,9 @@ static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, struct rockchip_pcie *pcie = &ep->rockchip; u32 r = rockchip_ob_region(addr); + if (test_bit(r, &ep->ob_region_map)) + return -EBUSY; + rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, addr, pci_addr, size); set_bit(r, &ep->ob_region_map); @@ -249,13 +292,9 @@ static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); struct rockchip_pcie *rockchip = &ep->rockchip; - u32 r; - - for (r = 0; r < ep->max_regions; r++) - if (ep->ob_addr[r] == addr) - break; + u32 r = rockchip_ob_region(addr); - if (r == ep->max_regions) + if (addr != ep->ob_addr[r] || !test_bit(r, &ep->ob_region_map)) return; rockchip_pcie_clear_ep_ob_atu(rockchip, r); @@ -351,9 +390,10 @@ static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, { struct rockchip_pcie *rockchip = &ep->rockchip; u32 flags, mme, data, data_mask; + size_t irq_pci_size, offset; + u64 irq_pci_addr; u8 msi_count; u64 pci_addr; - u32 r; /* Check MSI enable bit */ flags = rockchip_pcie_read(&ep->rockchip, @@ -389,18 +429,21 @@ static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, PCI_MSI_ADDRESS_LO); /* Set the outbound region if needed. */ - if (unlikely(ep->irq_pci_addr != (pci_addr & PCIE_ADDR_MASK) || + irq_pci_size = ~PCIE_ADDR_MASK + 1; + irq_pci_addr = rockchip_pcie_ep_align_addr(ep->epc, + pci_addr & PCIE_ADDR_MASK, + &irq_pci_size, &offset); + if (unlikely(ep->irq_pci_addr != irq_pci_addr || ep->irq_pci_fn != fn)) { - r = rockchip_ob_region(ep->irq_phys_addr); - rockchip_pcie_prog_ep_ob_atu(rockchip, fn, r, - ep->irq_phys_addr, - pci_addr & PCIE_ADDR_MASK, - ~PCIE_ADDR_MASK + 1); - ep->irq_pci_addr = (pci_addr & PCIE_ADDR_MASK); + rockchip_pcie_prog_ep_ob_atu(rockchip, fn, + rockchip_ob_region(ep->irq_phys_addr), + ep->irq_phys_addr, + irq_pci_addr, irq_pci_size); + ep->irq_pci_addr = irq_pci_addr; ep->irq_pci_fn = fn; } - writew(data, ep->irq_cpu_addr + (pci_addr & ~PCIE_ADDR_MASK)); + writew(data, ep->irq_cpu_addr + offset + (pci_addr & ~PCIE_ADDR_MASK)); return 0; } @@ -432,14 +475,222 @@ static int rockchip_pcie_ep_start(struct pci_epc *epc) rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG); + if (rockchip->perst_gpio) + enable_irq(ep->perst_irq); + + /* Enable configuration and start link training */ + rockchip_pcie_write(rockchip, + PCIE_CLIENT_LINK_TRAIN_ENABLE | + PCIE_CLIENT_CONF_ENABLE, + PCIE_CLIENT_CONFIG); + + if (!rockchip->perst_gpio) + schedule_delayed_work(&ep->link_training, 0); + + return 0; +} + +static void rockchip_pcie_ep_stop(struct pci_epc *epc) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + + if (rockchip->perst_gpio) { + ep->perst_asserted = true; + disable_irq(ep->perst_irq); + } + + cancel_delayed_work_sync(&ep->link_training); + + /* Stop link training and disable configuration */ + rockchip_pcie_write(rockchip, + PCIE_CLIENT_CONF_DISABLE | + PCIE_CLIENT_LINK_TRAIN_DISABLE, + PCIE_CLIENT_CONFIG); +} + +static void rockchip_pcie_ep_retrain_link(struct rockchip_pcie *rockchip) +{ + u32 status; + + status = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_LCS); + status |= PCI_EXP_LNKCTL_RL; + rockchip_pcie_write(rockchip, status, PCIE_EP_CONFIG_LCS); +} + +static bool rockchip_pcie_ep_link_up(struct rockchip_pcie *rockchip) +{ + u32 val = rockchip_pcie_read(rockchip, PCIE_CLIENT_BASIC_STATUS1); + + return PCIE_LINK_UP(val); +} + +static void rockchip_pcie_ep_link_training(struct work_struct *work) +{ + struct rockchip_pcie_ep *ep = + container_of(work, struct rockchip_pcie_ep, link_training.work); + struct rockchip_pcie *rockchip = &ep->rockchip; + struct device *dev = rockchip->dev; + u32 val; + int ret; + + /* Enable Gen1 training and wait for its completion */ + ret = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, + val, PCIE_LINK_TRAINING_DONE(val), 50, + LINK_TRAIN_TIMEOUT); + if (ret) + goto again; + + /* Make sure that the link is up */ + ret = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, + val, PCIE_LINK_UP(val), 50, + LINK_TRAIN_TIMEOUT); + if (ret) + goto again; + + /* + * Check the current speed: if gen2 speed was requested and we are not + * at gen2 speed yet, retrain again for gen2. + */ + val = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); + if (!PCIE_LINK_IS_GEN2(val) && rockchip->link_gen == 2) { + /* Enable retrain for gen2 */ + rockchip_pcie_ep_retrain_link(rockchip); + readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, + val, PCIE_LINK_IS_GEN2(val), 50, + LINK_TRAIN_TIMEOUT); + } + + /* Check again that the link is up */ + if (!rockchip_pcie_ep_link_up(rockchip)) + goto again; + + /* + * If PERST# was asserted while polling the link, do not notify + * the function. + */ + if (ep->perst_asserted) + return; + + val = rockchip_pcie_read(rockchip, PCIE_CLIENT_BASIC_STATUS0); + dev_info(dev, + "link up (negotiated speed: %sGT/s, width: x%lu)\n", + (val & PCIE_CLIENT_NEG_LINK_SPEED) ? "5" : "2.5", + ((val & PCIE_CLIENT_NEG_LINK_WIDTH_MASK) >> + PCIE_CLIENT_NEG_LINK_WIDTH_SHIFT) << 1); + + /* Notify the function */ + pci_epc_linkup(ep->epc); + ep->link_up = true; + + return; + +again: + schedule_delayed_work(&ep->link_training, msecs_to_jiffies(5)); +} + +static void rockchip_pcie_ep_perst_assert(struct rockchip_pcie_ep *ep) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + + dev_dbg(rockchip->dev, "PERST# asserted, link down\n"); + + if (ep->perst_asserted) + return; + + ep->perst_asserted = true; + + cancel_delayed_work_sync(&ep->link_training); + + if (ep->link_up) { + pci_epc_linkdown(ep->epc); + ep->link_up = false; + } +} + +static void rockchip_pcie_ep_perst_deassert(struct rockchip_pcie_ep *ep) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + + dev_dbg(rockchip->dev, "PERST# de-asserted, starting link training\n"); + + if (!ep->perst_asserted) + return; + + ep->perst_asserted = false; + + /* Enable link re-training */ + rockchip_pcie_ep_retrain_link(rockchip); + + /* Start link training */ + schedule_delayed_work(&ep->link_training, 0); +} + +static irqreturn_t rockchip_pcie_ep_perst_irq_thread(int irq, void *data) +{ + struct pci_epc *epc = data; + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + u32 perst = gpiod_get_value(rockchip->perst_gpio); + + if (perst) + rockchip_pcie_ep_perst_assert(ep); + else + rockchip_pcie_ep_perst_deassert(ep); + + irq_set_irq_type(ep->perst_irq, + (perst ? IRQF_TRIGGER_HIGH : IRQF_TRIGGER_LOW)); + + return IRQ_HANDLED; +} + +static int rockchip_pcie_ep_setup_irq(struct pci_epc *epc) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + struct device *dev = rockchip->dev; + int ret; + + if (!rockchip->perst_gpio) + return 0; + + /* PCIe reset interrupt */ + ep->perst_irq = gpiod_to_irq(rockchip->perst_gpio); + if (ep->perst_irq < 0) { + dev_err(dev, + "failed to get IRQ for PERST# GPIO: %d\n", + ep->perst_irq); + + return ep->perst_irq; + } + + /* + * The perst_gpio is active low, so when it is inactive on start, it + * is high and will trigger the perst_irq handler. So treat this initial + * IRQ as a dummy one by faking the host asserting PERST#. + */ + ep->perst_asserted = true; + irq_set_status_flags(ep->perst_irq, IRQ_NOAUTOEN); + ret = devm_request_threaded_irq(dev, ep->perst_irq, NULL, + rockchip_pcie_ep_perst_irq_thread, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + "pcie-ep-perst", epc); + if (ret) { + dev_err(dev, + "failed to request IRQ for PERST# GPIO: %d\n", + ret); + + return ret; + } + return 0; } static const struct pci_epc_features rockchip_pcie_epc_features = { - .linkup_notifier = false, + .linkup_notifier = true, .msi_capable = true, .msix_capable = false, - .align = 256, + .align = ROCKCHIP_PCIE_AT_SIZE_ALIGN, }; static const struct pci_epc_features* @@ -452,17 +703,19 @@ static const struct pci_epc_ops rockchip_pcie_epc_ops = { .write_header = rockchip_pcie_ep_write_header, .set_bar = rockchip_pcie_ep_set_bar, .clear_bar = rockchip_pcie_ep_clear_bar, + .align_addr = rockchip_pcie_ep_align_addr, .map_addr = rockchip_pcie_ep_map_addr, .unmap_addr = rockchip_pcie_ep_unmap_addr, .set_msi = rockchip_pcie_ep_set_msi, .get_msi = rockchip_pcie_ep_get_msi, .raise_irq = rockchip_pcie_ep_raise_irq, .start = rockchip_pcie_ep_start, + .stop = rockchip_pcie_ep_stop, .get_features = rockchip_pcie_ep_get_features, }; -static int rockchip_pcie_parse_ep_dt(struct rockchip_pcie *rockchip, - struct rockchip_pcie_ep *ep) +static int rockchip_pcie_ep_get_resources(struct rockchip_pcie *rockchip, + struct rockchip_pcie_ep *ep) { struct device *dev = rockchip->dev; int err; @@ -496,91 +749,63 @@ static const struct of_device_id rockchip_pcie_ep_of_match[] = { {}, }; -static int rockchip_pcie_ep_probe(struct platform_device *pdev) +static int rockchip_pcie_ep_init_ob_mem(struct rockchip_pcie_ep *ep) { - struct device *dev = &pdev->dev; - struct rockchip_pcie_ep *ep; - struct rockchip_pcie *rockchip; - struct pci_epc *epc; - size_t max_regions; + struct rockchip_pcie *rockchip = &ep->rockchip; + struct device *dev = rockchip->dev; struct pci_epc_mem_window *windows = NULL; int err, i; - u32 cfg_msi, cfg_msix_cp; - - ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); - if (!ep) - return -ENOMEM; - rockchip = &ep->rockchip; - rockchip->is_rc = false; - rockchip->dev = dev; - - epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); - if (IS_ERR(epc)) { - dev_err(dev, "failed to create epc device\n"); - return PTR_ERR(epc); - } - - ep->epc = epc; - epc_set_drvdata(epc, ep); - - err = rockchip_pcie_parse_ep_dt(rockchip, ep); - if (err) - return err; - - err = rockchip_pcie_enable_clocks(rockchip); - if (err) - return err; - - err = rockchip_pcie_init_port(rockchip); - if (err) - goto err_disable_clocks; - - /* Establish the link automatically */ - rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, - PCIE_CLIENT_CONFIG); - - max_regions = ep->max_regions; - ep->ob_addr = devm_kcalloc(dev, max_regions, sizeof(*ep->ob_addr), + ep->ob_addr = devm_kcalloc(dev, ep->max_regions, sizeof(*ep->ob_addr), GFP_KERNEL); - if (!ep->ob_addr) { - err = -ENOMEM; - goto err_uninit_port; - } - - /* Only enable function 0 by default */ - rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); + if (!ep->ob_addr) + return -ENOMEM; windows = devm_kcalloc(dev, ep->max_regions, sizeof(struct pci_epc_mem_window), GFP_KERNEL); - if (!windows) { - err = -ENOMEM; - goto err_uninit_port; - } + if (!windows) + return -ENOMEM; + for (i = 0; i < ep->max_regions; i++) { windows[i].phys_base = rockchip->mem_res->start + (SZ_1M * i); windows[i].size = SZ_1M; windows[i].page_size = SZ_1M; } - err = pci_epc_multi_mem_init(epc, windows, ep->max_regions); + err = pci_epc_multi_mem_init(ep->epc, windows, ep->max_regions); devm_kfree(dev, windows); if (err < 0) { dev_err(dev, "failed to initialize the memory space\n"); - goto err_uninit_port; + return err; } - ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr, + ep->irq_cpu_addr = pci_epc_mem_alloc_addr(ep->epc, &ep->irq_phys_addr, SZ_1M); if (!ep->irq_cpu_addr) { dev_err(dev, "failed to reserve memory space for MSI\n"); - err = -ENOMEM; goto err_epc_mem_exit; } ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR; + return 0; + +err_epc_mem_exit: + pci_epc_mem_exit(ep->epc); + + return err; +} + +static void rockchip_pcie_ep_exit_ob_mem(struct rockchip_pcie_ep *ep) +{ + pci_epc_mem_exit(ep->epc); +} + +static void rockchip_pcie_ep_hide_broken_msix_cap(struct rockchip_pcie *rockchip) +{ + u32 cfg_msi, cfg_msix_cp; + /* * MSI-X is not supported but the controller still advertises the MSI-X * capability by default, which can lead to the Root Complex side @@ -603,19 +828,68 @@ static int rockchip_pcie_ep_probe(struct platform_device *pdev) rockchip_pcie_write(rockchip, cfg_msi, PCIE_EP_CONFIG_BASE + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); +} - rockchip_pcie_write(rockchip, PCIE_CLIENT_CONF_ENABLE, - PCIE_CLIENT_CONFIG); +static int rockchip_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_pcie_ep *ep; + struct rockchip_pcie *rockchip; + struct pci_epc *epc; + int err; + + ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); + if (!ep) + return -ENOMEM; + + rockchip = &ep->rockchip; + rockchip->is_rc = false; + rockchip->dev = dev; + INIT_DELAYED_WORK(&ep->link_training, rockchip_pcie_ep_link_training); + + epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); + if (IS_ERR(epc)) { + dev_err(dev, "failed to create EPC device\n"); + return PTR_ERR(epc); + } + + ep->epc = epc; + epc_set_drvdata(epc, ep); + + err = rockchip_pcie_ep_get_resources(rockchip, ep); + if (err) + return err; + + err = rockchip_pcie_ep_init_ob_mem(ep); + if (err) + return err; + + err = rockchip_pcie_enable_clocks(rockchip); + if (err) + goto err_exit_ob_mem; + + err = rockchip_pcie_init_port(rockchip); + if (err) + goto err_disable_clocks; + + rockchip_pcie_ep_hide_broken_msix_cap(rockchip); + + /* Only enable function 0 by default */ + rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); pci_epc_init_notify(epc); + err = rockchip_pcie_ep_setup_irq(epc); + if (err < 0) + goto err_uninit_port; + return 0; -err_epc_mem_exit: - pci_epc_mem_exit(epc); err_uninit_port: rockchip_pcie_deinit_phys(rockchip); err_disable_clocks: rockchip_pcie_disable_clocks(rockchip); +err_exit_ob_mem: + rockchip_pcie_ep_exit_ob_mem(ep); return err; } diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c index cbec71114825..5adac6adc046 100644 --- a/drivers/pci/controller/pcie-rockchip-host.c +++ b/drivers/pci/controller/pcie-rockchip-host.c @@ -294,7 +294,7 @@ static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) int err, i = MAX_LANE_NUM; u32 status; - gpiod_set_value_cansleep(rockchip->ep_gpio, 0); + gpiod_set_value_cansleep(rockchip->perst_gpio, 0); err = rockchip_pcie_init_port(rockchip); if (err) @@ -323,7 +323,7 @@ static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) PCIE_CLIENT_CONFIG); msleep(PCIE_T_PVPERL_MS); - gpiod_set_value_cansleep(rockchip->ep_gpio, 1); + gpiod_set_value_cansleep(rockchip->perst_gpio, 1); msleep(PCIE_T_RRS_READY_MS); @@ -1050,7 +1050,7 @@ static struct platform_driver rockchip_pcie_driver = { .pm = &rockchip_pcie_pm_ops, }, .probe = rockchip_pcie_probe, - .remove_new = rockchip_pcie_remove, + .remove = rockchip_pcie_remove, }; module_platform_driver(rockchip_pcie_driver); diff --git a/drivers/pci/controller/pcie-rockchip.c b/drivers/pci/controller/pcie-rockchip.c index c07d7129f1c7..b9ade7632e11 100644 --- a/drivers/pci/controller/pcie-rockchip.c +++ b/drivers/pci/controller/pcie-rockchip.c @@ -119,13 +119,15 @@ int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip) return PTR_ERR(rockchip->aclk_rst); } - if (rockchip->is_rc) { - rockchip->ep_gpio = devm_gpiod_get_optional(dev, "ep", - GPIOD_OUT_LOW); - if (IS_ERR(rockchip->ep_gpio)) - return dev_err_probe(dev, PTR_ERR(rockchip->ep_gpio), - "failed to get ep GPIO\n"); - } + if (rockchip->is_rc) + rockchip->perst_gpio = devm_gpiod_get_optional(dev, "ep", + GPIOD_OUT_LOW); + else + rockchip->perst_gpio = devm_gpiod_get_optional(dev, "reset", + GPIOD_IN); + if (IS_ERR(rockchip->perst_gpio)) + return dev_err_probe(dev, PTR_ERR(rockchip->perst_gpio), + "failed to get PERST# GPIO\n"); rockchip->aclk_pcie = devm_clk_get(dev, "aclk"); if (IS_ERR(rockchip->aclk_pcie)) { @@ -244,11 +246,12 @@ int rockchip_pcie_init_port(struct rockchip_pcie *rockchip) rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1, PCIE_CLIENT_CONFIG); - regs = PCIE_CLIENT_LINK_TRAIN_ENABLE | PCIE_CLIENT_ARI_ENABLE | + regs = PCIE_CLIENT_ARI_ENABLE | PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes); if (rockchip->is_rc) - regs |= PCIE_CLIENT_CONF_ENABLE | PCIE_CLIENT_MODE_RC; + regs |= PCIE_CLIENT_LINK_TRAIN_ENABLE | + PCIE_CLIENT_CONF_ENABLE | PCIE_CLIENT_MODE_RC; else regs |= PCIE_CLIENT_CONF_DISABLE | PCIE_CLIENT_MODE_EP; diff --git a/drivers/pci/controller/pcie-rockchip.h b/drivers/pci/controller/pcie-rockchip.h index 6111de35f84c..a51b087ce878 100644 --- a/drivers/pci/controller/pcie-rockchip.h +++ b/drivers/pci/controller/pcie-rockchip.h @@ -26,12 +26,14 @@ #define MAX_LANE_NUM 4 #define MAX_REGION_LIMIT 32 #define MIN_EP_APERTURE 28 +#define LINK_TRAIN_TIMEOUT (500 * USEC_PER_MSEC) #define PCIE_CLIENT_BASE 0x0 #define PCIE_CLIENT_CONFIG (PCIE_CLIENT_BASE + 0x00) #define PCIE_CLIENT_CONF_ENABLE HIWORD_UPDATE_BIT(0x0001) #define PCIE_CLIENT_CONF_DISABLE HIWORD_UPDATE(0x0001, 0) #define PCIE_CLIENT_LINK_TRAIN_ENABLE HIWORD_UPDATE_BIT(0x0002) +#define PCIE_CLIENT_LINK_TRAIN_DISABLE HIWORD_UPDATE(0x0002, 0) #define PCIE_CLIENT_ARI_ENABLE HIWORD_UPDATE_BIT(0x0008) #define PCIE_CLIENT_CONF_LANE_NUM(x) HIWORD_UPDATE(0x0030, ENCODE_LANES(x)) #define PCIE_CLIENT_MODE_RC HIWORD_UPDATE_BIT(0x0040) @@ -49,6 +51,10 @@ #define PCIE_CLIENT_DEBUG_LTSSM_MASK GENMASK(5, 0) #define PCIE_CLIENT_DEBUG_LTSSM_L1 0x18 #define PCIE_CLIENT_DEBUG_LTSSM_L2 0x19 +#define PCIE_CLIENT_BASIC_STATUS0 (PCIE_CLIENT_BASE + 0x44) +#define PCIE_CLIENT_NEG_LINK_WIDTH_MASK GENMASK(7, 6) +#define PCIE_CLIENT_NEG_LINK_WIDTH_SHIFT 6 +#define PCIE_CLIENT_NEG_LINK_SPEED BIT(5) #define PCIE_CLIENT_BASIC_STATUS1 (PCIE_CLIENT_BASE + 0x48) #define PCIE_CLIENT_LINK_STATUS_UP 0x00300000 #define PCIE_CLIENT_LINK_STATUS_MASK 0x00300000 @@ -86,6 +92,8 @@ #define PCIE_CORE_CTRL_MGMT_BASE 0x900000 #define PCIE_CORE_CTRL (PCIE_CORE_CTRL_MGMT_BASE + 0x000) +#define PCIE_CORE_PL_CONF_LS_MASK 0x00000001 +#define PCIE_CORE_PL_CONF_LS_READY 0x00000001 #define PCIE_CORE_PL_CONF_SPEED_5G 0x00000008 #define PCIE_CORE_PL_CONF_SPEED_MASK 0x00000018 #define PCIE_CORE_PL_CONF_LANE_MASK 0x00000006 @@ -143,6 +151,7 @@ #define PCIE_RC_CONFIG_BASE 0xa00000 #define PCIE_EP_CONFIG_BASE 0xa00000 #define PCIE_EP_CONFIG_DID_VID (PCIE_EP_CONFIG_BASE + 0x00) +#define PCIE_EP_CONFIG_LCS (PCIE_EP_CONFIG_BASE + 0xd0) #define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08) #define PCIE_RC_CONFIG_DCR (PCIE_RC_CONFIG_BASE + 0xc4) #define PCIE_RC_CONFIG_DCR_CSPL_SHIFT 18 @@ -154,6 +163,7 @@ #define PCIE_RC_CONFIG_LINK_CAP (PCIE_RC_CONFIG_BASE + 0xcc) #define PCIE_RC_CONFIG_LINK_CAP_L0S BIT(10) #define PCIE_RC_CONFIG_LCS (PCIE_RC_CONFIG_BASE + 0xd0) +#define PCIE_EP_CONFIG_LCS (PCIE_EP_CONFIG_BASE + 0xd0) #define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c) #define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274) #define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20) @@ -191,6 +201,8 @@ #define ROCKCHIP_VENDOR_ID 0x1d87 #define PCIE_LINK_IS_L2(x) \ (((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2) +#define PCIE_LINK_TRAINING_DONE(x) \ + (((x) & PCIE_CORE_PL_CONF_LS_MASK) == PCIE_CORE_PL_CONF_LS_READY) #define PCIE_LINK_UP(x) \ (((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP) #define PCIE_LINK_IS_GEN2(x) \ @@ -241,10 +253,20 @@ #define ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK GENMASK(15, 8) #define ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR 0x1 #define ROCKCHIP_PCIE_EP_PCI_LEGACY_IRQ_ADDR 0x3 + +#define ROCKCHIP_PCIE_AT_MIN_NUM_BITS 8 +#define ROCKCHIP_PCIE_AT_MAX_NUM_BITS 20 +#define ROCKCHIP_PCIE_AT_SIZE_ALIGN (1UL << ROCKCHIP_PCIE_AT_MIN_NUM_BITS) + #define ROCKCHIP_PCIE_EP_FUNC_BASE(fn) \ (PCIE_EP_PF_CONFIG_REGS_BASE + (((fn) << 12) & GENMASK(19, 12))) #define ROCKCHIP_PCIE_EP_VIRT_FUNC_BASE(fn) \ (PCIE_EP_PF_CONFIG_REGS_BASE + 0x10000 + (((fn) << 12) & GENMASK(19, 12))) + +#define ROCKCHIP_PCIE_AT_MIN_NUM_BITS 8 +#define ROCKCHIP_PCIE_AT_MAX_NUM_BITS 20 +#define ROCKCHIP_PCIE_AT_SIZE_ALIGN (1UL << ROCKCHIP_PCIE_AT_MIN_NUM_BITS) + #define ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar) \ (PCIE_CORE_AXI_CONF_BASE + 0x0828 + (fn) * 0x0040 + (bar) * 0x0008) #define ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar) \ @@ -307,7 +329,7 @@ struct rockchip_pcie { struct regulator *vpcie3v3; /* 3.3V power supply */ struct regulator *vpcie1v8; /* 1.8V power supply */ struct regulator *vpcie0v9; /* 0.9V power supply */ - struct gpio_desc *ep_gpio; + struct gpio_desc *perst_gpio; u32 lanes; u8 lanes_map; int link_gen; diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index a8ae14474dd0..8d6e2a89b067 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -916,6 +916,6 @@ static struct platform_driver nwl_pcie_driver = { .of_match_table = nwl_pcie_of_match, }, .probe = nwl_pcie_probe, - .remove_new = nwl_pcie_remove, + .remove = nwl_pcie_remove, }; builtin_platform_driver(nwl_pcie_driver); diff --git a/drivers/pci/controller/plda/pcie-microchip-host.c b/drivers/pci/controller/plda/pcie-microchip-host.c index 48f60a04b740..6630cacef301 100644 --- a/drivers/pci/controller/plda/pcie-microchip-host.c +++ b/drivers/pci/controller/plda/pcie-microchip-host.c @@ -25,9 +25,6 @@ #define MC_PCIE1_BRIDGE_ADDR 0x00008000u #define MC_PCIE1_CTRL_ADDR 0x0000a000u -#define MC_PCIE_BRIDGE_ADDR (MC_PCIE1_BRIDGE_ADDR) -#define MC_PCIE_CTRL_ADDR (MC_PCIE1_CTRL_ADDR) - /* PCIe Controller Phy Regs */ #define SEC_ERROR_EVENT_CNT 0x20 #define DED_ERROR_EVENT_CNT 0x24 @@ -128,7 +125,6 @@ [EVENT_LOCAL_ ## x] = { __stringify(x), s } #define PCIE_EVENT(x) \ - .base = MC_PCIE_CTRL_ADDR, \ .offset = PCIE_EVENT_INT, \ .mask_offset = PCIE_EVENT_INT, \ .mask_high = 1, \ @@ -136,7 +132,6 @@ .enb_mask = PCIE_EVENT_INT_ENB_MASK #define SEC_EVENT(x) \ - .base = MC_PCIE_CTRL_ADDR, \ .offset = SEC_ERROR_INT, \ .mask_offset = SEC_ERROR_INT_MASK, \ .mask = SEC_ERROR_INT_ ## x ## _INT, \ @@ -144,7 +139,6 @@ .enb_mask = 0 #define DED_EVENT(x) \ - .base = MC_PCIE_CTRL_ADDR, \ .offset = DED_ERROR_INT, \ .mask_offset = DED_ERROR_INT_MASK, \ .mask_high = 1, \ @@ -152,7 +146,6 @@ .enb_mask = 0 #define LOCAL_EVENT(x) \ - .base = MC_PCIE_BRIDGE_ADDR, \ .offset = ISTATUS_LOCAL, \ .mask_offset = IMASK_LOCAL, \ .mask_high = 0, \ @@ -179,7 +172,8 @@ struct event_map { struct mc_pcie { struct plda_pcie_rp plda; - void __iomem *axi_base_addr; + void __iomem *bridge_base_addr; + void __iomem *ctrl_base_addr; }; struct cause { @@ -253,7 +247,6 @@ static struct event_map local_status_to_event[] = { }; static struct { - u32 base; u32 offset; u32 mask; u32 shift; @@ -325,8 +318,7 @@ static inline u32 reg_to_event(u32 reg, struct event_map field) static u32 pcie_events(struct mc_pcie *port) { - void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; - u32 reg = readl_relaxed(ctrl_base_addr + PCIE_EVENT_INT); + u32 reg = readl_relaxed(port->ctrl_base_addr + PCIE_EVENT_INT); u32 val = 0; int i; @@ -338,8 +330,7 @@ static u32 pcie_events(struct mc_pcie *port) static u32 sec_errors(struct mc_pcie *port) { - void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; - u32 reg = readl_relaxed(ctrl_base_addr + SEC_ERROR_INT); + u32 reg = readl_relaxed(port->ctrl_base_addr + SEC_ERROR_INT); u32 val = 0; int i; @@ -351,8 +342,7 @@ static u32 sec_errors(struct mc_pcie *port) static u32 ded_errors(struct mc_pcie *port) { - void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; - u32 reg = readl_relaxed(ctrl_base_addr + DED_ERROR_INT); + u32 reg = readl_relaxed(port->ctrl_base_addr + DED_ERROR_INT); u32 val = 0; int i; @@ -364,8 +354,7 @@ static u32 ded_errors(struct mc_pcie *port) static u32 local_events(struct mc_pcie *port) { - void __iomem *bridge_base_addr = port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; - u32 reg = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + u32 reg = readl_relaxed(port->bridge_base_addr + ISTATUS_LOCAL); u32 val = 0; int i; @@ -412,8 +401,12 @@ static void mc_ack_event_irq(struct irq_data *data) void __iomem *addr; u32 mask; - addr = mc_port->axi_base_addr + event_descs[event].base + - event_descs[event].offset; + if (event_descs[event].offset == ISTATUS_LOCAL) + addr = mc_port->bridge_base_addr; + else + addr = mc_port->ctrl_base_addr; + + addr += event_descs[event].offset; mask = event_descs[event].mask; mask |= event_descs[event].enb_mask; @@ -429,8 +422,12 @@ static void mc_mask_event_irq(struct irq_data *data) u32 mask; u32 val; - addr = mc_port->axi_base_addr + event_descs[event].base + - event_descs[event].mask_offset; + if (event_descs[event].offset == ISTATUS_LOCAL) + addr = mc_port->bridge_base_addr; + else + addr = mc_port->ctrl_base_addr; + + addr += event_descs[event].mask_offset; mask = event_descs[event].mask; if (event_descs[event].enb_mask) { mask <<= PCIE_EVENT_INT_ENB_SHIFT; @@ -460,8 +457,12 @@ static void mc_unmask_event_irq(struct irq_data *data) u32 mask; u32 val; - addr = mc_port->axi_base_addr + event_descs[event].base + - event_descs[event].mask_offset; + if (event_descs[event].offset == ISTATUS_LOCAL) + addr = mc_port->bridge_base_addr; + else + addr = mc_port->ctrl_base_addr; + + addr += event_descs[event].mask_offset; mask = event_descs[event].mask; if (event_descs[event].enb_mask) @@ -554,26 +555,20 @@ static const struct plda_event mc_event = { static inline void mc_clear_secs(struct mc_pcie *port) { - void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; - - writel_relaxed(SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT, ctrl_base_addr + - SEC_ERROR_INT); - writel_relaxed(0, ctrl_base_addr + SEC_ERROR_EVENT_CNT); + writel_relaxed(SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT, + port->ctrl_base_addr + SEC_ERROR_INT); + writel_relaxed(0, port->ctrl_base_addr + SEC_ERROR_EVENT_CNT); } static inline void mc_clear_deds(struct mc_pcie *port) { - void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; - - writel_relaxed(DED_ERROR_INT_ALL_RAM_DED_ERR_INT, ctrl_base_addr + - DED_ERROR_INT); - writel_relaxed(0, ctrl_base_addr + DED_ERROR_EVENT_CNT); + writel_relaxed(DED_ERROR_INT_ALL_RAM_DED_ERR_INT, + port->ctrl_base_addr + DED_ERROR_INT); + writel_relaxed(0, port->ctrl_base_addr + DED_ERROR_EVENT_CNT); } static void mc_disable_interrupts(struct mc_pcie *port) { - void __iomem *bridge_base_addr = port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; - void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; u32 val; /* Ensure ECC bypass is enabled */ @@ -581,22 +576,22 @@ static void mc_disable_interrupts(struct mc_pcie *port) ECC_CONTROL_RX_RAM_ECC_BYPASS | ECC_CONTROL_PCIE2AXI_RAM_ECC_BYPASS | ECC_CONTROL_AXI2PCIE_RAM_ECC_BYPASS; - writel_relaxed(val, ctrl_base_addr + ECC_CONTROL); + writel_relaxed(val, port->ctrl_base_addr + ECC_CONTROL); /* Disable SEC errors and clear any outstanding */ - writel_relaxed(SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT, ctrl_base_addr + - SEC_ERROR_INT_MASK); + writel_relaxed(SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT, + port->ctrl_base_addr + SEC_ERROR_INT_MASK); mc_clear_secs(port); /* Disable DED errors and clear any outstanding */ - writel_relaxed(DED_ERROR_INT_ALL_RAM_DED_ERR_INT, ctrl_base_addr + - DED_ERROR_INT_MASK); + writel_relaxed(DED_ERROR_INT_ALL_RAM_DED_ERR_INT, + port->ctrl_base_addr + DED_ERROR_INT_MASK); mc_clear_deds(port); /* Disable local interrupts and clear any outstanding */ - writel_relaxed(0, bridge_base_addr + IMASK_LOCAL); - writel_relaxed(GENMASK(31, 0), bridge_base_addr + ISTATUS_LOCAL); - writel_relaxed(GENMASK(31, 0), bridge_base_addr + ISTATUS_MSI); + writel_relaxed(0, port->bridge_base_addr + IMASK_LOCAL); + writel_relaxed(GENMASK(31, 0), port->bridge_base_addr + ISTATUS_LOCAL); + writel_relaxed(GENMASK(31, 0), port->bridge_base_addr + ISTATUS_MSI); /* Disable PCIe events and clear any outstanding */ val = PCIE_EVENT_INT_L2_EXIT_INT | @@ -605,11 +600,11 @@ static void mc_disable_interrupts(struct mc_pcie *port) PCIE_EVENT_INT_L2_EXIT_INT_MASK | PCIE_EVENT_INT_HOTRST_EXIT_INT_MASK | PCIE_EVENT_INT_DLUP_EXIT_INT_MASK; - writel_relaxed(val, ctrl_base_addr + PCIE_EVENT_INT); + writel_relaxed(val, port->ctrl_base_addr + PCIE_EVENT_INT); /* Disable host interrupts and clear any outstanding */ - writel_relaxed(0, bridge_base_addr + IMASK_HOST); - writel_relaxed(GENMASK(31, 0), bridge_base_addr + ISTATUS_HOST); + writel_relaxed(0, port->bridge_base_addr + IMASK_HOST); + writel_relaxed(GENMASK(31, 0), port->bridge_base_addr + ISTATUS_HOST); } static int mc_platform_init(struct pci_config_window *cfg) @@ -617,12 +612,10 @@ static int mc_platform_init(struct pci_config_window *cfg) struct device *dev = cfg->parent; struct platform_device *pdev = to_platform_device(dev); struct pci_host_bridge *bridge = platform_get_drvdata(pdev); - void __iomem *bridge_base_addr = - port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; int ret; /* Configure address translation table 0 for PCIe config space */ - plda_pcie_setup_window(bridge_base_addr, 0, cfg->res.start, + plda_pcie_setup_window(port->bridge_base_addr, 0, cfg->res.start, cfg->res.start, resource_size(&cfg->res)); @@ -649,7 +642,7 @@ static int mc_platform_init(struct pci_config_window *cfg) static int mc_host_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - void __iomem *bridge_base_addr; + void __iomem *apb_base_addr; struct plda_pcie_rp *plda; int ret; u32 val; @@ -661,30 +654,45 @@ static int mc_host_probe(struct platform_device *pdev) plda = &port->plda; plda->dev = dev; - port->axi_base_addr = devm_platform_ioremap_resource(pdev, 1); - if (IS_ERR(port->axi_base_addr)) - return PTR_ERR(port->axi_base_addr); + port->bridge_base_addr = devm_platform_ioremap_resource_byname(pdev, + "bridge"); + port->ctrl_base_addr = devm_platform_ioremap_resource_byname(pdev, + "ctrl"); + if (!IS_ERR(port->bridge_base_addr) && !IS_ERR(port->ctrl_base_addr)) + goto addrs_set; + + /* + * The original, incorrect, binding that lumped the control and + * bridge addresses together still needs to be handled by the driver. + */ + apb_base_addr = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(apb_base_addr)) + return dev_err_probe(dev, PTR_ERR(apb_base_addr), + "both legacy apb register and ctrl/bridge regions missing"); + + port->bridge_base_addr = apb_base_addr + MC_PCIE1_BRIDGE_ADDR; + port->ctrl_base_addr = apb_base_addr + MC_PCIE1_CTRL_ADDR; +addrs_set: mc_disable_interrupts(port); - bridge_base_addr = port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; - plda->bridge_addr = bridge_base_addr; + plda->bridge_addr = port->bridge_base_addr; plda->num_events = NUM_EVENTS; /* Allow enabling MSI by disabling MSI-X */ - val = readl(bridge_base_addr + PCIE_PCI_IRQ_DW0); + val = readl(port->bridge_base_addr + PCIE_PCI_IRQ_DW0); val &= ~MSIX_CAP_MASK; - writel(val, bridge_base_addr + PCIE_PCI_IRQ_DW0); + writel(val, port->bridge_base_addr + PCIE_PCI_IRQ_DW0); /* Pick num vectors from bitfile programmed onto FPGA fabric */ - val = readl(bridge_base_addr + PCIE_PCI_IRQ_DW0); + val = readl(port->bridge_base_addr + PCIE_PCI_IRQ_DW0); val &= NUM_MSI_MSGS_MASK; val >>= NUM_MSI_MSGS_SHIFT; plda->msi.num_vectors = 1 << val; /* Pick vector address from design */ - plda->msi.vector_phy = readl_relaxed(bridge_base_addr + IMSI_ADDR); + plda->msi.vector_phy = readl_relaxed(port->bridge_base_addr + IMSI_ADDR); ret = mc_pcie_init_clks(dev); if (ret) { diff --git a/drivers/pci/controller/plda/pcie-starfive.c b/drivers/pci/controller/plda/pcie-starfive.c index c9933ecf6833..e73c1b7bc8ef 100644 --- a/drivers/pci/controller/plda/pcie-starfive.c +++ b/drivers/pci/controller/plda/pcie-starfive.c @@ -404,6 +404,9 @@ static int starfive_pcie_probe(struct platform_device *pdev) if (ret) return ret; + pm_runtime_enable(&pdev->dev); + pm_runtime_get_sync(&pdev->dev); + plda->host_ops = &sf_host_ops; plda->num_events = PLDA_MAX_EVENT_NUM; /* mask doorbell event */ @@ -413,11 +416,12 @@ static int starfive_pcie_probe(struct platform_device *pdev) plda->events_bitmap <<= PLDA_NUM_DMA_EVENTS; ret = plda_pcie_host_init(&pcie->plda, &starfive_pcie_ops, &stf_pcie_event); - if (ret) + if (ret) { + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); return ret; + } - pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); platform_set_drvdata(pdev, pcie); return 0; @@ -480,7 +484,7 @@ static struct platform_driver starfive_pcie_driver = { .pm = pm_sleep_ptr(&starfive_pcie_pm_ops), }, .probe = starfive_pcie_probe, - .remove_new = starfive_pcie_remove, + .remove = starfive_pcie_remove, }; module_platform_driver(starfive_pcie_driver); diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index 264a180403a0..9d9596947350 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -740,11 +740,9 @@ static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata) if (!(features & VMD_FEAT_BIOS_PM_QUIRK)) return 0; - pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL); - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_LTR); if (!pos) - return 0; + goto out_state_change; /* * Skip if the max snoop LTR is non-zero, indicating BIOS has set it @@ -752,7 +750,7 @@ static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata) */ pci_read_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, <r_reg); if (!!(ltr_reg & (PCI_LTR_VALUE_MASK | PCI_LTR_SCALE_MASK))) - return 0; + goto out_state_change; /* * Set the default values to the maximum required by the platform to @@ -764,6 +762,13 @@ static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata) pci_write_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, ltr_reg); pci_info(pdev, "VMD: Default LTR value set by driver\n"); +out_state_change: + /* + * Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per + * PCIe r6.0, sec 5.5.4. + */ + pci_set_power_state_locked(pdev, PCI_D0); + pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL); return 0; } @@ -1100,6 +1105,10 @@ static const struct pci_device_id vmd_ids[] = { .driver_data = VMD_FEATS_CLIENT,}, {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B), .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xb60b), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xb06f), + .driver_data = VMD_FEATS_CLIENT,}, {0,} }; MODULE_DEVICE_TABLE(pci, vmd_ids); diff --git a/drivers/pci/devres.c b/drivers/pci/devres.c index b133967faef8..3b59a86a764b 100644 --- a/drivers/pci/devres.c +++ b/drivers/pci/devres.c @@ -773,7 +773,7 @@ EXPORT_SYMBOL(pcim_iomap_region); * Unmap a BAR and release its region manually. Only pass BARs that were * previously mapped by pcim_iomap_region(). */ -static void pcim_iounmap_region(struct pci_dev *pdev, int bar) +void pcim_iounmap_region(struct pci_dev *pdev, int bar) { struct pcim_addr_devres res_searched; @@ -784,6 +784,7 @@ static void pcim_iounmap_region(struct pci_dev *pdev, int bar) devres_release(&pdev->dev, pcim_addr_resource_release, pcim_addr_resources_match, &res_searched); } +EXPORT_SYMBOL(pcim_iounmap_region); /** * pcim_iomap_regions - Request and iomap PCI BARs (DEPRECATED) @@ -939,7 +940,7 @@ static void pcim_release_all_regions(struct pci_dev *pdev) * desired, release individual regions with pcim_release_region() or all of * them at once with pcim_release_all_regions(). */ -static int pcim_request_all_regions(struct pci_dev *pdev, const char *name) +int pcim_request_all_regions(struct pci_dev *pdev, const char *name) { int ret; int bar; @@ -957,69 +958,17 @@ err: return ret; } +EXPORT_SYMBOL(pcim_request_all_regions); /** - * pcim_iomap_regions_request_all - Request all BARs and iomap specified ones - * (DEPRECATED) - * @pdev: PCI device to map IO resources for - * @mask: Mask of BARs to iomap - * @name: Name associated with the requests - * - * Returns: 0 on success, negative error code on failure. - * - * Request all PCI BARs and iomap regions specified by @mask. - * - * To release these resources manually, call pcim_release_region() for the - * regions and pcim_iounmap() for the mappings. - * - * This function is DEPRECATED. Don't use it in new code. Instead, use one - * of the pcim_* region request functions in combination with a pcim_* - * mapping function. - */ -int pcim_iomap_regions_request_all(struct pci_dev *pdev, int mask, - const char *name) -{ - int bar; - int ret; - void __iomem **legacy_iomap_table; - - ret = pcim_request_all_regions(pdev, name); - if (ret != 0) - return ret; - - for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { - if (!mask_contains_bar(mask, bar)) - continue; - if (!pcim_iomap(pdev, bar, 0)) - goto err; - } - - return 0; - -err: - /* - * If bar is larger than 0, then pcim_iomap() above has most likely - * failed because of -EINVAL. If it is equal 0, most likely the table - * couldn't be created, indicating -ENOMEM. - */ - ret = bar > 0 ? -EINVAL : -ENOMEM; - legacy_iomap_table = (void __iomem **)pcim_iomap_table(pdev); - - while (--bar >= 0) - pcim_iounmap(pdev, legacy_iomap_table[bar]); - - pcim_release_all_regions(pdev); - - return ret; -} -EXPORT_SYMBOL(pcim_iomap_regions_request_all); - -/** - * pcim_iounmap_regions - Unmap and release PCI BARs + * pcim_iounmap_regions - Unmap and release PCI BARs (DEPRECATED) * @pdev: PCI device to map IO resources for * @mask: Mask of BARs to unmap and release * * Unmap and release regions specified by @mask. + * + * This function is DEPRECATED. Do not use it in new code. + * Use pcim_iounmap_region() instead. */ void pcim_iounmap_regions(struct pci_dev *pdev, int mask) { diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index 652d63df9d22..7bd7892c5222 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -146,6 +146,7 @@ static int pci_doe_send_req(struct pci_doe_mb *doe_mb, { struct pci_dev *pdev = doe_mb->pdev; int offset = doe_mb->cap_offset; + unsigned long timeout_jiffies; size_t length, remainder; u32 val; int i; @@ -155,8 +156,19 @@ static int pci_doe_send_req(struct pci_doe_mb *doe_mb, * someone other than Linux (e.g. firmware) is using the mailbox. Note * it is expected that firmware and OS will negotiate access rights via * an, as yet to be defined, method. + * + * Wait up to one PCI_DOE_TIMEOUT period to allow the prior command to + * finish. Otherwise, simply error out as unable to field the request. + * + * PCIe r6.2 sec 6.30.3 states no interrupt is raised when the DOE Busy + * bit is cleared, so polling here is our best option for the moment. */ - pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; + do { + pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); + } while (FIELD_GET(PCI_DOE_STATUS_BUSY, val) && + !time_after(jiffies, timeout_jiffies)); + if (FIELD_GET(PCI_DOE_STATUS_BUSY, val)) return -EBUSY; diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c index 1c40d2506aef..260b7de2dbd5 100644 --- a/drivers/pci/ecam.c +++ b/drivers/pci/ecam.c @@ -55,7 +55,7 @@ struct pci_config_window *pci_ecam_create(struct device *dev, bus_range_max = resource_size(cfgres) >> bus_shift; if (bus_range > bus_range_max) { bus_range = bus_range_max; - cfg->busr.end = busr->start + bus_range - 1; + resource_set_size(&cfg->busr, bus_range); dev_warn(dev, "ECAM area %pR can only accommodate %pR (reduced from %pR desired)\n", cfgres, &cfg->busr, busr); } diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c index 7d070b1def11..54286a40bdfb 100644 --- a/drivers/pci/endpoint/functions/pci-epf-mhi.c +++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c @@ -867,12 +867,18 @@ static int pci_epf_mhi_bind(struct pci_epf *epf) { struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; + struct device *dev = &epf->dev; struct platform_device *pdev = to_platform_device(epc->dev.parent); struct resource *res; int ret; /* Get MMIO base address from Endpoint controller */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mmio"); + if (!res) { + dev_err(dev, "Failed to get \"mmio\" resource\n"); + return -ENODEV; + } + epf_mhi->mmio_phys = res->start; epf_mhi->mmio_size = resource_size(res); diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 7c2ed6eae53a..ef6677f34116 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -291,8 +291,6 @@ static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test) dma_release_channel(epf_test->dma_chan_rx); epf_test->dma_chan_rx = NULL; - - return; } static void pci_epf_test_print_rate(struct pci_epf_test *epf_test, @@ -317,91 +315,92 @@ static void pci_epf_test_print_rate(struct pci_epf_test *epf_test, static void pci_epf_test_copy(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) { - int ret; - void __iomem *src_addr; - void __iomem *dst_addr; - phys_addr_t src_phys_addr; - phys_addr_t dst_phys_addr; + int ret = 0; struct timespec64 start, end; struct pci_epf *epf = epf_test->epf; - struct device *dev = &epf->dev; struct pci_epc *epc = epf->epc; + struct device *dev = &epf->dev; + struct pci_epc_map src_map, dst_map; + u64 src_addr = reg->src_addr; + u64 dst_addr = reg->dst_addr; + size_t copy_size = reg->size; + ssize_t map_size = 0; + void *copy_buf = NULL, *buf; - src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size); - if (!src_addr) { - dev_err(dev, "Failed to allocate source address\n"); - reg->status = STATUS_SRC_ADDR_INVALID; - ret = -ENOMEM; - goto err; - } - - ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr, - reg->src_addr, reg->size); - if (ret) { - dev_err(dev, "Failed to map source address\n"); - reg->status = STATUS_SRC_ADDR_INVALID; - goto err_src_addr; - } - - dst_addr = pci_epc_mem_alloc_addr(epc, &dst_phys_addr, reg->size); - if (!dst_addr) { - dev_err(dev, "Failed to allocate destination address\n"); - reg->status = STATUS_DST_ADDR_INVALID; - ret = -ENOMEM; - goto err_src_map_addr; - } - - ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr, - reg->dst_addr, reg->size); - if (ret) { - dev_err(dev, "Failed to map destination address\n"); - reg->status = STATUS_DST_ADDR_INVALID; - goto err_dst_addr; - } - - ktime_get_ts64(&start); if (reg->flags & FLAG_USE_DMA) { if (epf_test->dma_private) { dev_err(dev, "Cannot transfer data using DMA\n"); ret = -EINVAL; - goto err_map_addr; + goto set_status; } - - ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr, - src_phys_addr, reg->size, 0, - DMA_MEM_TO_MEM); - if (ret) - dev_err(dev, "Data transfer failed\n"); } else { - void *buf; - - buf = kzalloc(reg->size, GFP_KERNEL); - if (!buf) { + copy_buf = kzalloc(copy_size, GFP_KERNEL); + if (!copy_buf) { ret = -ENOMEM; - goto err_map_addr; + goto set_status; } - - memcpy_fromio(buf, src_addr, reg->size); - memcpy_toio(dst_addr, buf, reg->size); - kfree(buf); + buf = copy_buf; } - ktime_get_ts64(&end); - pci_epf_test_print_rate(epf_test, "COPY", reg->size, &start, &end, - reg->flags & FLAG_USE_DMA); -err_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr); + while (copy_size) { + ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, + src_addr, copy_size, &src_map); + if (ret) { + dev_err(dev, "Failed to map source address\n"); + reg->status = STATUS_SRC_ADDR_INVALID; + goto free_buf; + } + + ret = pci_epc_mem_map(epf->epc, epf->func_no, epf->vfunc_no, + dst_addr, copy_size, &dst_map); + if (ret) { + dev_err(dev, "Failed to map destination address\n"); + reg->status = STATUS_DST_ADDR_INVALID; + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, + &src_map); + goto free_buf; + } -err_dst_addr: - pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size); + map_size = min_t(size_t, dst_map.pci_size, src_map.pci_size); -err_src_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr); + ktime_get_ts64(&start); + if (reg->flags & FLAG_USE_DMA) { + ret = pci_epf_test_data_transfer(epf_test, + dst_map.phys_addr, src_map.phys_addr, + map_size, 0, DMA_MEM_TO_MEM); + if (ret) { + dev_err(dev, "Data transfer failed\n"); + goto unmap; + } + } else { + memcpy_fromio(buf, src_map.virt_addr, map_size); + memcpy_toio(dst_map.virt_addr, buf, map_size); + buf += map_size; + } + ktime_get_ts64(&end); -err_src_addr: - pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size); + copy_size -= map_size; + src_addr += map_size; + dst_addr += map_size; -err: + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); + map_size = 0; + } + + pci_epf_test_print_rate(epf_test, "COPY", reg->size, &start, + &end, reg->flags & FLAG_USE_DMA); + +unmap: + if (map_size) { + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); + } + +free_buf: + kfree(copy_buf); + +set_status: if (!ret) reg->status |= STATUS_COPY_SUCCESS; else @@ -411,82 +410,89 @@ err: static void pci_epf_test_read(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) { - int ret; - void __iomem *src_addr; - void *buf; + int ret = 0; + void *src_buf, *buf; u32 crc32; - phys_addr_t phys_addr; + struct pci_epc_map map; phys_addr_t dst_phys_addr; struct timespec64 start, end; struct pci_epf *epf = epf_test->epf; - struct device *dev = &epf->dev; struct pci_epc *epc = epf->epc; + struct device *dev = &epf->dev; struct device *dma_dev = epf->epc->dev.parent; + u64 src_addr = reg->src_addr; + size_t src_size = reg->size; + ssize_t map_size = 0; - src_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size); - if (!src_addr) { - dev_err(dev, "Failed to allocate address\n"); - reg->status = STATUS_SRC_ADDR_INVALID; + src_buf = kzalloc(src_size, GFP_KERNEL); + if (!src_buf) { ret = -ENOMEM; - goto err; + goto set_status; } + buf = src_buf; - ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr, - reg->src_addr, reg->size); - if (ret) { - dev_err(dev, "Failed to map address\n"); - reg->status = STATUS_SRC_ADDR_INVALID; - goto err_addr; - } - - buf = kzalloc(reg->size, GFP_KERNEL); - if (!buf) { - ret = -ENOMEM; - goto err_map_addr; - } + while (src_size) { + ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, + src_addr, src_size, &map); + if (ret) { + dev_err(dev, "Failed to map address\n"); + reg->status = STATUS_SRC_ADDR_INVALID; + goto free_buf; + } - if (reg->flags & FLAG_USE_DMA) { - dst_phys_addr = dma_map_single(dma_dev, buf, reg->size, - DMA_FROM_DEVICE); - if (dma_mapping_error(dma_dev, dst_phys_addr)) { - dev_err(dev, "Failed to map destination buffer addr\n"); - ret = -ENOMEM; - goto err_dma_map; + map_size = map.pci_size; + if (reg->flags & FLAG_USE_DMA) { + dst_phys_addr = dma_map_single(dma_dev, buf, map_size, + DMA_FROM_DEVICE); + if (dma_mapping_error(dma_dev, dst_phys_addr)) { + dev_err(dev, + "Failed to map destination buffer addr\n"); + ret = -ENOMEM; + goto unmap; + } + + ktime_get_ts64(&start); + ret = pci_epf_test_data_transfer(epf_test, + dst_phys_addr, map.phys_addr, + map_size, src_addr, DMA_DEV_TO_MEM); + if (ret) + dev_err(dev, "Data transfer failed\n"); + ktime_get_ts64(&end); + + dma_unmap_single(dma_dev, dst_phys_addr, map_size, + DMA_FROM_DEVICE); + + if (ret) + goto unmap; + } else { + ktime_get_ts64(&start); + memcpy_fromio(buf, map.virt_addr, map_size); + ktime_get_ts64(&end); } - ktime_get_ts64(&start); - ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr, - phys_addr, reg->size, - reg->src_addr, DMA_DEV_TO_MEM); - if (ret) - dev_err(dev, "Data transfer failed\n"); - ktime_get_ts64(&end); + src_size -= map_size; + src_addr += map_size; + buf += map_size; - dma_unmap_single(dma_dev, dst_phys_addr, reg->size, - DMA_FROM_DEVICE); - } else { - ktime_get_ts64(&start); - memcpy_fromio(buf, src_addr, reg->size); - ktime_get_ts64(&end); + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); + map_size = 0; } - pci_epf_test_print_rate(epf_test, "READ", reg->size, &start, &end, - reg->flags & FLAG_USE_DMA); + pci_epf_test_print_rate(epf_test, "READ", reg->size, &start, + &end, reg->flags & FLAG_USE_DMA); - crc32 = crc32_le(~0, buf, reg->size); + crc32 = crc32_le(~0, src_buf, reg->size); if (crc32 != reg->checksum) ret = -EIO; -err_dma_map: - kfree(buf); - -err_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr); +unmap: + if (map_size) + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); -err_addr: - pci_epc_mem_free_addr(epc, phys_addr, src_addr, reg->size); +free_buf: + kfree(src_buf); -err: +set_status: if (!ret) reg->status |= STATUS_READ_SUCCESS; else @@ -496,71 +502,79 @@ err: static void pci_epf_test_write(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) { - int ret; - void __iomem *dst_addr; - void *buf; - phys_addr_t phys_addr; + int ret = 0; + void *dst_buf, *buf; + struct pci_epc_map map; phys_addr_t src_phys_addr; struct timespec64 start, end; struct pci_epf *epf = epf_test->epf; - struct device *dev = &epf->dev; struct pci_epc *epc = epf->epc; + struct device *dev = &epf->dev; struct device *dma_dev = epf->epc->dev.parent; + u64 dst_addr = reg->dst_addr; + size_t dst_size = reg->size; + ssize_t map_size = 0; - dst_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size); - if (!dst_addr) { - dev_err(dev, "Failed to allocate address\n"); - reg->status = STATUS_DST_ADDR_INVALID; + dst_buf = kzalloc(dst_size, GFP_KERNEL); + if (!dst_buf) { ret = -ENOMEM; - goto err; + goto set_status; } + get_random_bytes(dst_buf, dst_size); + reg->checksum = crc32_le(~0, dst_buf, dst_size); + buf = dst_buf; - ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr, - reg->dst_addr, reg->size); - if (ret) { - dev_err(dev, "Failed to map address\n"); - reg->status = STATUS_DST_ADDR_INVALID; - goto err_addr; - } - - buf = kzalloc(reg->size, GFP_KERNEL); - if (!buf) { - ret = -ENOMEM; - goto err_map_addr; - } - - get_random_bytes(buf, reg->size); - reg->checksum = crc32_le(~0, buf, reg->size); - - if (reg->flags & FLAG_USE_DMA) { - src_phys_addr = dma_map_single(dma_dev, buf, reg->size, - DMA_TO_DEVICE); - if (dma_mapping_error(dma_dev, src_phys_addr)) { - dev_err(dev, "Failed to map source buffer addr\n"); - ret = -ENOMEM; - goto err_dma_map; + while (dst_size) { + ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, + dst_addr, dst_size, &map); + if (ret) { + dev_err(dev, "Failed to map address\n"); + reg->status = STATUS_DST_ADDR_INVALID; + goto free_buf; } - ktime_get_ts64(&start); + map_size = map.pci_size; + if (reg->flags & FLAG_USE_DMA) { + src_phys_addr = dma_map_single(dma_dev, buf, map_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, src_phys_addr)) { + dev_err(dev, + "Failed to map source buffer addr\n"); + ret = -ENOMEM; + goto unmap; + } + + ktime_get_ts64(&start); + + ret = pci_epf_test_data_transfer(epf_test, + map.phys_addr, src_phys_addr, + map_size, dst_addr, + DMA_MEM_TO_DEV); + if (ret) + dev_err(dev, "Data transfer failed\n"); + ktime_get_ts64(&end); + + dma_unmap_single(dma_dev, src_phys_addr, map_size, + DMA_TO_DEVICE); + + if (ret) + goto unmap; + } else { + ktime_get_ts64(&start); + memcpy_toio(map.virt_addr, buf, map_size); + ktime_get_ts64(&end); + } - ret = pci_epf_test_data_transfer(epf_test, phys_addr, - src_phys_addr, reg->size, - reg->dst_addr, - DMA_MEM_TO_DEV); - if (ret) - dev_err(dev, "Data transfer failed\n"); - ktime_get_ts64(&end); + dst_size -= map_size; + dst_addr += map_size; + buf += map_size; - dma_unmap_single(dma_dev, src_phys_addr, reg->size, - DMA_TO_DEVICE); - } else { - ktime_get_ts64(&start); - memcpy_toio(dst_addr, buf, reg->size); - ktime_get_ts64(&end); + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); + map_size = 0; } - pci_epf_test_print_rate(epf_test, "WRITE", reg->size, &start, &end, - reg->flags & FLAG_USE_DMA); + pci_epf_test_print_rate(epf_test, "WRITE", reg->size, &start, + &end, reg->flags & FLAG_USE_DMA); /* * wait 1ms inorder for the write to complete. Without this delay L3 @@ -568,16 +582,14 @@ static void pci_epf_test_write(struct pci_epf_test *epf_test, */ usleep_range(1000, 2000); -err_dma_map: - kfree(buf); - -err_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr); +unmap: + if (map_size) + pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); -err_addr: - pci_epc_mem_free_addr(epc, phys_addr, dst_addr, reg->size); +free_buf: + kfree(dst_buf); -err: +set_status: if (!ret) reg->status |= STATUS_WRITE_SUCCESS; else @@ -786,7 +798,7 @@ static void pci_epf_test_epc_deinit(struct pci_epf *epf) { struct pci_epf_test *epf_test = epf_get_drvdata(epf); - cancel_delayed_work(&epf_test->cmd_handler); + cancel_delayed_work_sync(&epf_test->cmd_handler); pci_epf_test_clean_dma_chan(epf_test); pci_epf_test_clear_bar(epf); } @@ -917,7 +929,7 @@ static void pci_epf_test_unbind(struct pci_epf *epf) struct pci_epf_test *epf_test = epf_get_drvdata(epf); struct pci_epc *epc = epf->epc; - cancel_delayed_work(&epf_test->cmd_handler); + cancel_delayed_work_sync(&epf_test->cmd_handler); if (epc->init_complete) { pci_epf_test_clean_dma_chan(epf_test); pci_epf_test_clear_bar(epf); diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 17f007109255..bed7c7d1fe3c 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -128,6 +128,18 @@ enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features } EXPORT_SYMBOL_GPL(pci_epc_get_next_free_bar); +static bool pci_epc_function_is_valid(struct pci_epc *epc, + u8 func_no, u8 vfunc_no) +{ + if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) + return false; + + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return false; + + return true; +} + /** * pci_epc_get_features() - get the features supported by EPC * @epc: the features supported by *this* EPC device will be returned @@ -145,10 +157,7 @@ const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc, { const struct pci_epc_features *epc_features; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return NULL; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return NULL; if (!epc->ops->get_features) @@ -218,10 +227,7 @@ int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { int ret; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return -EINVAL; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return -EINVAL; if (!epc->ops->raise_irq) @@ -262,10 +268,7 @@ int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { int ret; - if (IS_ERR_OR_NULL(epc)) - return -EINVAL; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return -EINVAL; if (!epc->ops->map_msi_irq) @@ -293,10 +296,7 @@ int pci_epc_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { int interrupt; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return 0; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return 0; if (!epc->ops->get_msi) @@ -329,11 +329,10 @@ int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, u8 interrupts) int ret; u8 encode_int; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions || - interrupts < 1 || interrupts > 32) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return -EINVAL; - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (interrupts < 1 || interrupts > 32) return -EINVAL; if (!epc->ops->set_msi) @@ -361,10 +360,7 @@ int pci_epc_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { int interrupt; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return 0; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return 0; if (!epc->ops->get_msix) @@ -397,11 +393,10 @@ int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { int ret; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions || - interrupts < 1 || interrupts > 2048) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return -EINVAL; - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (interrupts < 1 || interrupts > 2048) return -EINVAL; if (!epc->ops->set_msix) @@ -428,10 +423,7 @@ EXPORT_SYMBOL_GPL(pci_epc_set_msix); void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, phys_addr_t phys_addr) { - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return; if (!epc->ops->unmap_addr) @@ -459,10 +451,7 @@ int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { int ret; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return -EINVAL; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return -EINVAL; if (!epc->ops->map_addr) @@ -478,6 +467,109 @@ int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, EXPORT_SYMBOL_GPL(pci_epc_map_addr); /** + * pci_epc_mem_map() - allocate and map a PCI address to a CPU address + * @epc: the EPC device on which the CPU address is to be allocated and mapped + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function + * @pci_addr: PCI address to which the CPU address should be mapped + * @pci_size: the number of bytes to map starting from @pci_addr + * @map: where to return the mapping information + * + * Allocate a controller memory address region and map it to a RC PCI address + * region, taking into account the controller physical address mapping + * constraints using the controller operation align_addr(). If this operation is + * not defined, we assume that there are no alignment constraints for the + * mapping. + * + * The effective size of the PCI address range mapped from @pci_addr is + * indicated by @map->pci_size. This size may be less than the requested + * @pci_size. The local virtual CPU address for the mapping is indicated by + * @map->virt_addr (@map->phys_addr indicates the physical address). + * The size and CPU address of the controller memory allocated and mapped are + * respectively indicated by @map->map_size and @map->virt_base (and + * @map->phys_base for the physical address of @map->virt_base). + * + * Returns 0 on success and a negative error code in case of error. + */ +int pci_epc_mem_map(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u64 pci_addr, size_t pci_size, struct pci_epc_map *map) +{ + size_t map_size = pci_size; + size_t map_offset = 0; + int ret; + + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) + return -EINVAL; + + if (!pci_size || !map) + return -EINVAL; + + /* + * Align the PCI address to map. If the controller defines the + * .align_addr() operation, use it to determine the PCI address to map + * and the size of the mapping. Otherwise, assume that the controller + * has no alignment constraint. + */ + memset(map, 0, sizeof(*map)); + map->pci_addr = pci_addr; + if (epc->ops->align_addr) + map->map_pci_addr = + epc->ops->align_addr(epc, pci_addr, + &map_size, &map_offset); + else + map->map_pci_addr = pci_addr; + map->map_size = map_size; + if (map->map_pci_addr + map->map_size < pci_addr + pci_size) + map->pci_size = map->map_pci_addr + map->map_size - pci_addr; + else + map->pci_size = pci_size; + + map->virt_base = pci_epc_mem_alloc_addr(epc, &map->phys_base, + map->map_size); + if (!map->virt_base) + return -ENOMEM; + + map->phys_addr = map->phys_base + map_offset; + map->virt_addr = map->virt_base + map_offset; + + ret = pci_epc_map_addr(epc, func_no, vfunc_no, map->phys_base, + map->map_pci_addr, map->map_size); + if (ret) { + pci_epc_mem_free_addr(epc, map->phys_base, map->virt_base, + map->map_size); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(pci_epc_mem_map); + +/** + * pci_epc_mem_unmap() - unmap and free a CPU address region + * @epc: the EPC device on which the CPU address is allocated and mapped + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function + * @map: the mapping information + * + * Unmap and free a CPU address region that was allocated and mapped with + * pci_epc_mem_map(). + */ +void pci_epc_mem_unmap(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_map *map) +{ + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) + return; + + if (!map || !map->virt_base) + return; + + pci_epc_unmap_addr(epc, func_no, vfunc_no, map->phys_base); + pci_epc_mem_free_addr(epc, map->phys_base, map->virt_base, + map->map_size); +} +EXPORT_SYMBOL_GPL(pci_epc_mem_unmap); + +/** * pci_epc_clear_bar() - reset the BAR * @epc: the EPC device for which the BAR has to be cleared * @func_no: the physical endpoint function number in the EPC device @@ -489,12 +581,11 @@ EXPORT_SYMBOL_GPL(pci_epc_map_addr); void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar) { - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions || - (epf_bar->barno == BAR_5 && - epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return; - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (epf_bar->barno == BAR_5 && + epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) return; if (!epc->ops->clear_bar) @@ -521,18 +612,16 @@ int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, int ret; int flags = epf_bar->flags; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions || - (epf_bar->barno == BAR_5 && - flags & PCI_BASE_ADDRESS_MEM_TYPE_64) || + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) + return -EINVAL; + + if ((epf_bar->barno == BAR_5 && flags & PCI_BASE_ADDRESS_MEM_TYPE_64) || (flags & PCI_BASE_ADDRESS_SPACE_IO && flags & PCI_BASE_ADDRESS_IO_MASK) || (upper_32_bits(epf_bar->size) && !(flags & PCI_BASE_ADDRESS_MEM_TYPE_64))) return -EINVAL; - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) - return -EINVAL; - if (!epc->ops->set_bar) return 0; @@ -561,10 +650,7 @@ int pci_epc_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { int ret; - if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) - return -EINVAL; - - if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + if (!pci_epc_function_is_valid(epc, func_no, vfunc_no)) return -EINVAL; /* Only Virtual Function #1 has deviceID */ @@ -660,18 +746,18 @@ void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf, if (IS_ERR_OR_NULL(epc) || !epf) return; + mutex_lock(&epc->list_lock); if (type == PRIMARY_INTERFACE) { func_no = epf->func_no; list = &epf->list; + epf->epc = NULL; } else { func_no = epf->sec_epc_func_no; list = &epf->sec_epc_list; + epf->sec_epc = NULL; } - - mutex_lock(&epc->list_lock); clear_bit(func_no, &epc->function_num_map); list_del(list); - epf->epc = NULL; mutex_unlock(&epc->list_lock); } EXPORT_SYMBOL_GPL(pci_epc_remove_epf); @@ -837,11 +923,10 @@ EXPORT_SYMBOL_GPL(pci_epc_bus_master_enable_notify); void pci_epc_destroy(struct pci_epc *epc) { pci_ep_cfs_remove_epc_group(epc->group); - device_unregister(&epc->dev); - #ifdef CONFIG_PCI_DOMAINS_GENERIC - pci_bus_release_domain_nr(&epc->dev, epc->domain_nr); + pci_bus_release_domain_nr(epc->dev.parent, epc->domain_nr); #endif + device_unregister(&epc->dev); } EXPORT_SYMBOL_GPL(pci_epc_destroy); diff --git a/drivers/pci/endpoint/pci-epc-mem.c b/drivers/pci/endpoint/pci-epc-mem.c index a9c028f58da1..218a60e945db 100644 --- a/drivers/pci/endpoint/pci-epc-mem.c +++ b/drivers/pci/endpoint/pci-epc-mem.c @@ -178,7 +178,7 @@ EXPORT_SYMBOL_GPL(pci_epc_mem_exit); void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc, phys_addr_t *phys_addr, size_t size) { - void __iomem *virt_addr = NULL; + void __iomem *virt_addr; struct pci_epc_mem *mem; unsigned int page_shift; size_t align_size; @@ -188,10 +188,13 @@ void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc, for (i = 0; i < epc->num_windows; i++) { mem = epc->windows[i]; - mutex_lock(&mem->lock); + if (size > mem->window.size) + continue; + align_size = ALIGN(size, mem->window.page_size); order = pci_epc_mem_get_order(mem, align_size); + mutex_lock(&mem->lock); pageno = bitmap_find_free_region(mem->bitmap, mem->pages, order); if (pageno >= 0) { @@ -211,7 +214,7 @@ void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc, mutex_unlock(&mem->lock); } - return virt_addr; + return NULL; } EXPORT_SYMBOL_GPL(pci_epc_mem_alloc_addr); diff --git a/drivers/pci/hotplug/Kconfig b/drivers/pci/hotplug/Kconfig index 1472aef0fb81..123c4c7c2ab5 100644 --- a/drivers/pci/hotplug/Kconfig +++ b/drivers/pci/hotplug/Kconfig @@ -118,6 +118,16 @@ config HOTPLUG_PCI_CPCI_GENERIC When in doubt, say N. +config HOTPLUG_PCI_OCTEONEP + bool "Marvell OCTEON PCI Hotplug driver" + depends on HOTPLUG_PCI + help + Say Y here if you have an OCTEON PCIe device with a hotplug + controller. This driver enables the non-controller functions of the + device to be registered as hotplug slots. + + When in doubt, say N. + config HOTPLUG_PCI_SHPC bool "SHPC PCI Hotplug driver" help diff --git a/drivers/pci/hotplug/Makefile b/drivers/pci/hotplug/Makefile index 240c99517d5e..40aaf31fe338 100644 --- a/drivers/pci/hotplug/Makefile +++ b/drivers/pci/hotplug/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_HOTPLUG_PCI_RPA) += rpaphp.o obj-$(CONFIG_HOTPLUG_PCI_RPA_DLPAR) += rpadlpar_io.o obj-$(CONFIG_HOTPLUG_PCI_ACPI) += acpiphp.o obj-$(CONFIG_HOTPLUG_PCI_S390) += s390_pci_hpc.o +obj-$(CONFIG_HOTPLUG_PCI_OCTEONEP) += octep_hp.o # acpiphp_ibm extends acpiphp, so should be linked afterwards. diff --git a/drivers/pci/hotplug/acpiphp_ampere_altra.c b/drivers/pci/hotplug/acpiphp_ampere_altra.c index f5c9e741c1d4..70dbc0431fc6 100644 --- a/drivers/pci/hotplug/acpiphp_ampere_altra.c +++ b/drivers/pci/hotplug/acpiphp_ampere_altra.c @@ -119,7 +119,7 @@ static struct platform_driver altra_led_driver = { .acpi_match_table = altra_led_ids, }, .probe = altra_led_probe, - .remove_new = altra_led_remove, + .remove = altra_led_remove, }; module_platform_driver(altra_led_driver); diff --git a/drivers/pci/hotplug/cpci_hotplug.h b/drivers/pci/hotplug/cpci_hotplug.h index 6d8970d8c3f2..03fa39ab0c88 100644 --- a/drivers/pci/hotplug/cpci_hotplug.h +++ b/drivers/pci/hotplug/cpci_hotplug.h @@ -44,7 +44,6 @@ struct cpci_hp_controller_ops { int (*enable_irq)(void); int (*disable_irq)(void); int (*check_irq)(void *dev_id); - int (*hardware_test)(struct slot *slot, u32 value); u8 (*get_power)(struct slot *slot); int (*set_power)(struct slot *slot, int value); }; diff --git a/drivers/pci/hotplug/cpqphp_pci.c b/drivers/pci/hotplug/cpqphp_pci.c index 718bc6cf12cb..ef7534a3ca40 100644 --- a/drivers/pci/hotplug/cpqphp_pci.c +++ b/drivers/pci/hotplug/cpqphp_pci.c @@ -12,8 +12,11 @@ * */ +#define pr_fmt(fmt) "cpqphp: " fmt + #include <linux/module.h> #include <linux/kernel.h> +#include <linux/printk.h> #include <linux/types.h> #include <linux/slab.h> #include <linux/workqueue.h> @@ -132,18 +135,6 @@ int cpqhp_unconfigure_device(struct pci_func *func) return 0; } -static int PCI_RefinedAccessConfig(struct pci_bus *bus, unsigned int devfn, u8 offset, u32 *value) -{ - u32 vendID = 0; - - if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, &vendID) == -1) - return -1; - if (PCI_POSSIBLE_ERROR(vendID)) - return -1; - return pci_bus_read_config_dword(bus, devfn, offset, value); -} - - /* * cpqhp_set_irq * @@ -202,13 +193,16 @@ static int PCI_ScanBusForNonBridge(struct controller *ctrl, u8 bus_num, u8 *dev_ { u16 tdevice; u32 work; - u8 tbus; + int ret = -1; ctrl->pci_bus->number = bus_num; for (tdevice = 0; tdevice < 0xFF; tdevice++) { /* Scan for access first */ - if (PCI_RefinedAccessConfig(ctrl->pci_bus, tdevice, 0x08, &work) == -1) + if (!pci_bus_read_dev_vendor_id(ctrl->pci_bus, tdevice, &work, 0)) + continue; + ret = pci_bus_read_config_dword(ctrl->pci_bus, tdevice, PCI_CLASS_REVISION, &work); + if (ret) continue; dbg("Looking for nonbridge bus_num %d dev_num %d\n", bus_num, tdevice); /* Yep we got one. Not a bridge ? */ @@ -216,23 +210,20 @@ static int PCI_ScanBusForNonBridge(struct controller *ctrl, u8 bus_num, u8 *dev_ *dev_num = tdevice; dbg("found it !\n"); return 0; - } - } - for (tdevice = 0; tdevice < 0xFF; tdevice++) { - /* Scan for access first */ - if (PCI_RefinedAccessConfig(ctrl->pci_bus, tdevice, 0x08, &work) == -1) - continue; - dbg("Looking for bridge bus_num %d dev_num %d\n", bus_num, tdevice); - /* Yep we got one. bridge ? */ - if ((work >> 8) == PCI_TO_PCI_BRIDGE_CLASS) { - pci_bus_read_config_byte(ctrl->pci_bus, PCI_DEVFN(tdevice, 0), PCI_SECONDARY_BUS, &tbus); - /* XXX: no recursion, wtf? */ - dbg("Recurse on bus_num %d tdevice %d\n", tbus, tdevice); - return 0; + } else { + /* + * XXX: Code whose debug printout indicated + * recursion to buses underneath bridges might be + * necessary was removed because it never did + * any recursion. + */ + ret = 0; + pr_warn("missing feature: bridge scan recursion not implemented\n"); } } - return -1; + + return ret; } diff --git a/drivers/pci/hotplug/cpqphp_sysfs.c b/drivers/pci/hotplug/cpqphp_sysfs.c index fed1360ee9b1..6143ebf71f21 100644 --- a/drivers/pci/hotplug/cpqphp_sysfs.c +++ b/drivers/pci/hotplug/cpqphp_sysfs.c @@ -123,7 +123,6 @@ static int spew_debug_info(struct controller *ctrl, char *data, int size) struct ctrl_dbg { int size; char *data; - struct controller *ctrl; }; #define MAX_OUTPUT (4*PAGE_SIZE) diff --git a/drivers/pci/hotplug/octep_hp.c b/drivers/pci/hotplug/octep_hp.c new file mode 100644 index 000000000000..2bce7296c050 --- /dev/null +++ b/drivers/pci/hotplug/octep_hp.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2024 Marvell. */ + +#include <linux/cleanup.h> +#include <linux/container_of.h> +#include <linux/delay.h> +#include <linux/dev_printk.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci_hotplug.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/workqueue.h> + +#define OCTEP_HP_INTR_OFFSET(x) (0x20400 + ((x) << 4)) +#define OCTEP_HP_INTR_VECTOR(x) (16 + (x)) +#define OCTEP_HP_DRV_NAME "octep_hp" + +/* + * Type of MSI-X interrupts. OCTEP_HP_INTR_VECTOR() and + * OCTEP_HP_INTR_OFFSET() generate the vector and offset for an interrupt + * type. + */ +enum octep_hp_intr_type { + OCTEP_HP_INTR_INVALID = -1, + OCTEP_HP_INTR_ENA = 0, + OCTEP_HP_INTR_DIS = 1, + OCTEP_HP_INTR_MAX = 2, +}; + +struct octep_hp_cmd { + struct list_head list; + enum octep_hp_intr_type intr_type; + u64 intr_val; +}; + +struct octep_hp_slot { + struct list_head list; + struct hotplug_slot slot; + u16 slot_number; + struct pci_dev *hp_pdev; + unsigned int hp_devfn; + struct octep_hp_controller *ctrl; +}; + +struct octep_hp_intr_info { + enum octep_hp_intr_type type; + int number; + char name[16]; +}; + +struct octep_hp_controller { + void __iomem *base; + struct pci_dev *pdev; + struct octep_hp_intr_info intr[OCTEP_HP_INTR_MAX]; + struct work_struct work; + struct list_head slot_list; + struct mutex slot_lock; /* Protects slot_list */ + struct list_head hp_cmd_list; + spinlock_t hp_cmd_lock; /* Protects hp_cmd_list */ +}; + +static void octep_hp_enable_pdev(struct octep_hp_controller *hp_ctrl, + struct octep_hp_slot *hp_slot) +{ + guard(mutex)(&hp_ctrl->slot_lock); + if (hp_slot->hp_pdev) { + pci_dbg(hp_slot->hp_pdev, "Slot %s is already enabled\n", + hotplug_slot_name(&hp_slot->slot)); + return; + } + + /* Scan the device and add it to the bus */ + hp_slot->hp_pdev = pci_scan_single_device(hp_ctrl->pdev->bus, + hp_slot->hp_devfn); + pci_bus_assign_resources(hp_ctrl->pdev->bus); + pci_bus_add_device(hp_slot->hp_pdev); + + dev_dbg(&hp_slot->hp_pdev->dev, "Enabled slot %s\n", + hotplug_slot_name(&hp_slot->slot)); +} + +static void octep_hp_disable_pdev(struct octep_hp_controller *hp_ctrl, + struct octep_hp_slot *hp_slot) +{ + guard(mutex)(&hp_ctrl->slot_lock); + if (!hp_slot->hp_pdev) { + pci_dbg(hp_ctrl->pdev, "Slot %s is already disabled\n", + hotplug_slot_name(&hp_slot->slot)); + return; + } + + pci_dbg(hp_slot->hp_pdev, "Disabling slot %s\n", + hotplug_slot_name(&hp_slot->slot)); + + /* Remove the device from the bus */ + pci_stop_and_remove_bus_device_locked(hp_slot->hp_pdev); + hp_slot->hp_pdev = NULL; +} + +static int octep_hp_enable_slot(struct hotplug_slot *slot) +{ + struct octep_hp_slot *hp_slot = + container_of(slot, struct octep_hp_slot, slot); + + octep_hp_enable_pdev(hp_slot->ctrl, hp_slot); + return 0; +} + +static int octep_hp_disable_slot(struct hotplug_slot *slot) +{ + struct octep_hp_slot *hp_slot = + container_of(slot, struct octep_hp_slot, slot); + + octep_hp_disable_pdev(hp_slot->ctrl, hp_slot); + return 0; +} + +static struct hotplug_slot_ops octep_hp_slot_ops = { + .enable_slot = octep_hp_enable_slot, + .disable_slot = octep_hp_disable_slot, +}; + +#define SLOT_NAME_SIZE 16 +static struct octep_hp_slot * +octep_hp_register_slot(struct octep_hp_controller *hp_ctrl, + struct pci_dev *pdev, u16 slot_number) +{ + char slot_name[SLOT_NAME_SIZE]; + struct octep_hp_slot *hp_slot; + int ret; + + hp_slot = kzalloc(sizeof(*hp_slot), GFP_KERNEL); + if (!hp_slot) + return ERR_PTR(-ENOMEM); + + hp_slot->ctrl = hp_ctrl; + hp_slot->hp_pdev = pdev; + hp_slot->hp_devfn = pdev->devfn; + hp_slot->slot_number = slot_number; + hp_slot->slot.ops = &octep_hp_slot_ops; + + snprintf(slot_name, sizeof(slot_name), "octep_hp_%u", slot_number); + ret = pci_hp_register(&hp_slot->slot, hp_ctrl->pdev->bus, + PCI_SLOT(pdev->devfn), slot_name); + if (ret) { + kfree(hp_slot); + return ERR_PTR(ret); + } + + pci_info(pdev, "Registered slot %s for device %s\n", + slot_name, pci_name(pdev)); + + list_add_tail(&hp_slot->list, &hp_ctrl->slot_list); + octep_hp_disable_pdev(hp_ctrl, hp_slot); + + return hp_slot; +} + +static void octep_hp_deregister_slot(void *data) +{ + struct octep_hp_slot *hp_slot = data; + struct octep_hp_controller *hp_ctrl = hp_slot->ctrl; + + pci_hp_deregister(&hp_slot->slot); + octep_hp_enable_pdev(hp_ctrl, hp_slot); + list_del(&hp_slot->list); + kfree(hp_slot); +} + +static const char *octep_hp_cmd_name(enum octep_hp_intr_type type) +{ + switch (type) { + case OCTEP_HP_INTR_ENA: + return "hotplug enable"; + case OCTEP_HP_INTR_DIS: + return "hotplug disable"; + default: + return "invalid"; + } +} + +static void octep_hp_cmd_handler(struct octep_hp_controller *hp_ctrl, + struct octep_hp_cmd *hp_cmd) +{ + struct octep_hp_slot *hp_slot; + + /* + * Enable or disable the slots based on the slot mask. + * intr_val is a bit mask where each bit represents a slot. + */ + list_for_each_entry(hp_slot, &hp_ctrl->slot_list, list) { + if (!(hp_cmd->intr_val & BIT(hp_slot->slot_number))) + continue; + + pci_info(hp_ctrl->pdev, "Received %s command for slot %s\n", + octep_hp_cmd_name(hp_cmd->intr_type), + hotplug_slot_name(&hp_slot->slot)); + + switch (hp_cmd->intr_type) { + case OCTEP_HP_INTR_ENA: + octep_hp_enable_pdev(hp_ctrl, hp_slot); + break; + case OCTEP_HP_INTR_DIS: + octep_hp_disable_pdev(hp_ctrl, hp_slot); + break; + default: + break; + } + } +} + +static void octep_hp_work_handler(struct work_struct *work) +{ + struct octep_hp_controller *hp_ctrl; + struct octep_hp_cmd *hp_cmd; + unsigned long flags; + + hp_ctrl = container_of(work, struct octep_hp_controller, work); + + /* Process all the hotplug commands */ + spin_lock_irqsave(&hp_ctrl->hp_cmd_lock, flags); + while (!list_empty(&hp_ctrl->hp_cmd_list)) { + hp_cmd = list_first_entry(&hp_ctrl->hp_cmd_list, + struct octep_hp_cmd, list); + list_del(&hp_cmd->list); + spin_unlock_irqrestore(&hp_ctrl->hp_cmd_lock, flags); + + octep_hp_cmd_handler(hp_ctrl, hp_cmd); + kfree(hp_cmd); + + spin_lock_irqsave(&hp_ctrl->hp_cmd_lock, flags); + } + spin_unlock_irqrestore(&hp_ctrl->hp_cmd_lock, flags); +} + +static enum octep_hp_intr_type octep_hp_intr_type(struct octep_hp_intr_info *intr, + int irq) +{ + enum octep_hp_intr_type type; + + for (type = OCTEP_HP_INTR_ENA; type < OCTEP_HP_INTR_MAX; type++) { + if (intr[type].number == irq) + return type; + } + + return OCTEP_HP_INTR_INVALID; +} + +static irqreturn_t octep_hp_intr_handler(int irq, void *data) +{ + struct octep_hp_controller *hp_ctrl = data; + struct pci_dev *pdev = hp_ctrl->pdev; + enum octep_hp_intr_type type; + struct octep_hp_cmd *hp_cmd; + u64 intr_val; + + type = octep_hp_intr_type(hp_ctrl->intr, irq); + if (type == OCTEP_HP_INTR_INVALID) { + pci_err(pdev, "Invalid interrupt %d\n", irq); + return IRQ_HANDLED; + } + + /* Read and clear the interrupt */ + intr_val = readq(hp_ctrl->base + OCTEP_HP_INTR_OFFSET(type)); + writeq(intr_val, hp_ctrl->base + OCTEP_HP_INTR_OFFSET(type)); + + hp_cmd = kzalloc(sizeof(*hp_cmd), GFP_ATOMIC); + if (!hp_cmd) + return IRQ_HANDLED; + + hp_cmd->intr_val = intr_val; + hp_cmd->intr_type = type; + + /* Add the command to the list and schedule the work */ + spin_lock(&hp_ctrl->hp_cmd_lock); + list_add_tail(&hp_cmd->list, &hp_ctrl->hp_cmd_list); + spin_unlock(&hp_ctrl->hp_cmd_lock); + schedule_work(&hp_ctrl->work); + + return IRQ_HANDLED; +} + +static void octep_hp_irq_cleanup(void *data) +{ + struct octep_hp_controller *hp_ctrl = data; + + pci_free_irq_vectors(hp_ctrl->pdev); + flush_work(&hp_ctrl->work); +} + +static int octep_hp_request_irq(struct octep_hp_controller *hp_ctrl, + enum octep_hp_intr_type type) +{ + struct pci_dev *pdev = hp_ctrl->pdev; + struct octep_hp_intr_info *intr; + int irq; + + irq = pci_irq_vector(pdev, OCTEP_HP_INTR_VECTOR(type)); + if (irq < 0) + return irq; + + intr = &hp_ctrl->intr[type]; + intr->number = irq; + intr->type = type; + snprintf(intr->name, sizeof(intr->name), "octep_hp_%d", type); + + return devm_request_irq(&pdev->dev, irq, octep_hp_intr_handler, + IRQF_SHARED, intr->name, hp_ctrl); +} + +static int octep_hp_controller_setup(struct pci_dev *pdev, + struct octep_hp_controller *hp_ctrl) +{ + struct device *dev = &pdev->dev; + enum octep_hp_intr_type type; + int ret; + + ret = pcim_enable_device(pdev); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable PCI device\n"); + + hp_ctrl->base = pcim_iomap_region(pdev, 0, OCTEP_HP_DRV_NAME); + if (IS_ERR(hp_ctrl->base)) + return dev_err_probe(dev, PTR_ERR(hp_ctrl->base), + "Failed to map PCI device region\n"); + + pci_set_master(pdev); + pci_set_drvdata(pdev, hp_ctrl); + + INIT_LIST_HEAD(&hp_ctrl->slot_list); + INIT_LIST_HEAD(&hp_ctrl->hp_cmd_list); + mutex_init(&hp_ctrl->slot_lock); + spin_lock_init(&hp_ctrl->hp_cmd_lock); + INIT_WORK(&hp_ctrl->work, octep_hp_work_handler); + hp_ctrl->pdev = pdev; + + ret = pci_alloc_irq_vectors(pdev, 1, + OCTEP_HP_INTR_VECTOR(OCTEP_HP_INTR_MAX), + PCI_IRQ_MSIX); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to alloc MSI-X vectors\n"); + + ret = devm_add_action(&pdev->dev, octep_hp_irq_cleanup, hp_ctrl); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to add IRQ cleanup action\n"); + + for (type = OCTEP_HP_INTR_ENA; type < OCTEP_HP_INTR_MAX; type++) { + ret = octep_hp_request_irq(hp_ctrl, type); + if (ret) + return dev_err_probe(dev, ret, + "Failed to request IRQ for vector %d\n", + OCTEP_HP_INTR_VECTOR(type)); + } + + return 0; +} + +static int octep_hp_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct octep_hp_controller *hp_ctrl; + struct pci_dev *tmp_pdev, *next; + struct octep_hp_slot *hp_slot; + u16 slot_number = 0; + int ret; + + hp_ctrl = devm_kzalloc(&pdev->dev, sizeof(*hp_ctrl), GFP_KERNEL); + if (!hp_ctrl) + return -ENOMEM; + + ret = octep_hp_controller_setup(pdev, hp_ctrl); + if (ret) + return ret; + + /* + * Register all hotplug slots. Hotplug controller is the first function + * of the PCI device. The hotplug slots are the remaining functions of + * the PCI device. The hotplug slot functions are logically removed from + * the bus during probing and are re-enabled by the driver when a + * hotplug event is received. + */ + list_for_each_entry_safe(tmp_pdev, next, &pdev->bus->devices, bus_list) { + if (tmp_pdev == pdev) + continue; + + hp_slot = octep_hp_register_slot(hp_ctrl, tmp_pdev, slot_number); + if (IS_ERR(hp_slot)) + return dev_err_probe(&pdev->dev, PTR_ERR(hp_slot), + "Failed to register hotplug slot %u\n", + slot_number); + + ret = devm_add_action(&pdev->dev, octep_hp_deregister_slot, + hp_slot); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to add action for deregistering slot %u\n", + slot_number); + slot_number++; + } + + return 0; +} + +#define PCI_DEVICE_ID_CAVIUM_OCTEP_HP_CTLR 0xa0e3 +static struct pci_device_id octep_hp_pci_map[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_OCTEP_HP_CTLR) }, + { }, +}; + +static struct pci_driver octep_hp = { + .name = OCTEP_HP_DRV_NAME, + .id_table = octep_hp_pci_map, + .probe = octep_hp_pci_probe, +}; + +module_pci_driver(octep_hp); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION("Marvell OCTEON PCI Hotplug driver"); diff --git a/drivers/pci/hotplug/pci_hotplug_core.c b/drivers/pci/hotplug/pci_hotplug_core.c index 058d5937d8a9..36236ac88fd5 100644 --- a/drivers/pci/hotplug/pci_hotplug_core.c +++ b/drivers/pci/hotplug/pci_hotplug_core.c @@ -388,8 +388,8 @@ static struct hotplug_slot *get_slot_from_name(const char *name) /** * __pci_hp_register - register a hotplug_slot with the PCI hotplug subsystem - * @bus: bus this slot is on * @slot: pointer to the &struct hotplug_slot to register + * @bus: bus this slot is on * @devnr: device number * @name: name registered with kobject core * @owner: caller module owner @@ -498,8 +498,6 @@ EXPORT_SYMBOL_GPL(pci_hp_add); * * The @slot must have been registered with the pci hotplug subsystem * previously with a call to pci_hp_register(). - * - * Returns 0 if successful, anything else for an error. */ void pci_hp_deregister(struct hotplug_slot *slot) { @@ -513,8 +511,6 @@ EXPORT_SYMBOL_GPL(pci_hp_deregister); * @slot: pointer to the &struct hotplug_slot to unpublish * * Remove a hotplug slot's sysfs interface. - * - * Returns 0 on success or a negative int on error. */ void pci_hp_del(struct hotplug_slot *slot) { @@ -545,8 +541,6 @@ EXPORT_SYMBOL_GPL(pci_hp_del); * the driver may no longer invoke hotplug_slot_name() to get the slot's * unique name. The driver no longer needs to handle a ->reset_slot callback * from this point on. - * - * Returns 0 on success or a negative int on error. */ void pci_hp_destroy(struct hotplug_slot *slot) { diff --git a/drivers/pci/hotplug/pciehp_ctrl.c b/drivers/pci/hotplug/pciehp_ctrl.c index dcdbfcf404dd..d603a7aa7483 100644 --- a/drivers/pci/hotplug/pciehp_ctrl.c +++ b/drivers/pci/hotplug/pciehp_ctrl.c @@ -19,6 +19,8 @@ #include <linux/types.h> #include <linux/pm_runtime.h> #include <linux/pci.h> + +#include "../pci.h" #include "pciehp.h" /* The following routines constitute the bulk of the @@ -127,6 +129,9 @@ static void remove_board(struct controller *ctrl, bool safe_removal) pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, INDICATOR_NOOP); + + /* Don't carry LBMS indications across */ + pcie_reset_lbms_count(ctrl->pcie->port); } static int pciehp_enable_slot(struct controller *ctrl); diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c index 736ad8baa2a5..bb5a8d9f03ad 100644 --- a/drivers/pci/hotplug/pciehp_hpc.c +++ b/drivers/pci/hotplug/pciehp_hpc.c @@ -319,7 +319,7 @@ int pciehp_check_link_status(struct controller *ctrl) return -1; } - pcie_update_link_speed(ctrl->pcie->port->subordinate, lnk_status); + __pcie_update_link_speed(ctrl->pcie->port->subordinate, lnk_status); if (!found) { ctrl_info(ctrl, "Slot(%s): No device found\n", diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index aaa33e8dc4c9..4be402fe9ab9 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -327,8 +327,8 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id) virtfn->resource[i].name = pci_name(virtfn); virtfn->resource[i].flags = res->flags; size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); - virtfn->resource[i].start = res->start + size * id; - virtfn->resource[i].end = virtfn->resource[i].start + size - 1; + resource_set_range(&virtfn->resource[i], + res->start + size * id, size); rc = request_resource(res, &virtfn->resource[i]); BUG_ON(rc); } @@ -804,7 +804,7 @@ found: goto failed; } iov->barsz[i] = resource_size(res); - res->end = res->start + resource_size(res) * total - 1; + resource_set_size(res, resource_size(res) * total); pci_info(dev, "%s %pR: contains BAR %d for %d VFs\n", res_name, res, i, total); i += bar64; diff --git a/drivers/pci/of.c b/drivers/pci/of.c index dacea3fc5128..52f770bcc481 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -728,6 +728,33 @@ out_free_name: } #endif +/** + * of_pci_supply_present() - Check if the power supply is present for the PCI + * device + * @np: Device tree node + * + * Check if the power supply for the PCI device is present in the device tree + * node or not. + * + * Return: true if at least one power supply exists; false otherwise. + */ +bool of_pci_supply_present(struct device_node *np) +{ + struct property *prop; + char *supply; + + if (!np) + return false; + + for_each_property_of_node(np, prop) { + supply = strrchr(prop->name, '-'); + if (supply && !strcmp(supply, "-supply")) + return true; + } + + return false; +} + #endif /* CONFIG_PCI */ /** diff --git a/drivers/pci/of_property.c b/drivers/pci/of_property.c index 5a0b98e69795..886c236e5de6 100644 --- a/drivers/pci/of_property.c +++ b/drivers/pci/of_property.c @@ -126,7 +126,7 @@ static int of_pci_prop_ranges(struct pci_dev *pdev, struct of_changeset *ocs, if (of_pci_get_addr_flags(&res[j], &flags)) continue; - val64 = res[j].start; + val64 = pci_bus_address(pdev, &res[j] - pdev->resource); of_pci_set_address(pdev, rp[i].parent_addr, val64, 0, flags, false); if (pci_is_bridge(pdev)) { diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 5d0f4db1cab7..3e5a117f5b5d 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -521,6 +521,31 @@ static ssize_t bus_rescan_store(struct device *dev, static struct device_attribute dev_attr_bus_rescan = __ATTR(rescan, 0200, NULL, bus_rescan_store); +static ssize_t reset_subordinate_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_bus *bus = pdev->subordinate; + unsigned long val; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) { + int ret = __pci_reset_bus(bus); + + if (ret) + return ret; + } + + return count; +} +static DEVICE_ATTR_WO(reset_subordinate); + #if defined(CONFIG_PM) && defined(CONFIG_ACPI) static ssize_t d3cold_allowed_store(struct device *dev, struct device_attribute *attr, @@ -625,6 +650,7 @@ static struct attribute *pci_dev_attrs[] = { static struct attribute *pci_bridge_attrs[] = { &dev_attr_subordinate_bus_number.attr, &dev_attr_secondary_bus_number.attr, + &dev_attr_reset_subordinate.attr, NULL, }; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 63690375d36b..0b29ec6e8e5e 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1832,6 +1832,7 @@ int pci_save_state(struct pci_dev *dev) pci_save_dpc_state(dev); pci_save_aer_state(dev); pci_save_ptm_state(dev); + pci_save_tph_state(dev); return pci_save_vc_state(dev); } EXPORT_SYMBOL(pci_save_state); @@ -1937,6 +1938,7 @@ void pci_restore_state(struct pci_dev *dev) pci_restore_rebar_state(dev); pci_restore_dpc_state(dev); pci_restore_ptm_state(dev); + pci_restore_tph_state(dev); pci_aer_clear_status(dev); pci_restore_aer_state(dev); @@ -4744,7 +4746,7 @@ int pcie_retrain_link(struct pci_dev *pdev, bool use_lt) * to track link speed or width changes made by hardware itself * in attempt to correct unreliable link operation. */ - pcie_capability_write_word(pdev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); + pcie_reset_lbms_count(pdev); return rc; } @@ -5162,6 +5164,8 @@ static void pci_dev_save_and_disable(struct pci_dev *dev) */ if (err_handler && err_handler->reset_prepare) err_handler->reset_prepare(dev); + else if (dev->driver) + pci_warn(dev, "resetting"); /* * Wake-up device prior to save. PM registers default to D0 after @@ -5195,6 +5199,8 @@ static void pci_dev_restore(struct pci_dev *dev) */ if (err_handler && err_handler->reset_done) err_handler->reset_done(dev); + else if (dev->driver) + pci_warn(dev, "reset done"); } /* dev->reset_methods[] is a 0-terminated list of indices into this array */ @@ -5248,7 +5254,7 @@ static ssize_t reset_method_store(struct device *dev, const char *buf, size_t count) { struct pci_dev *pdev = to_pci_dev(dev); - char *options, *name; + char *options, *tmp_options, *name; int m, n; u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 }; @@ -5268,7 +5274,8 @@ static ssize_t reset_method_store(struct device *dev, return -ENOMEM; n = 0; - while ((name = strsep(&options, " ")) != NULL) { + tmp_options = options; + while ((name = strsep(&tmp_options, " ")) != NULL) { if (sysfs_streq(name, "")) continue; @@ -5884,7 +5891,7 @@ EXPORT_SYMBOL_GPL(pci_probe_reset_bus); * * Same as above except return -EAGAIN if the bus cannot be locked */ -static int __pci_reset_bus(struct pci_bus *bus) +int __pci_reset_bus(struct pci_bus *bus) { int rc; @@ -6193,38 +6200,64 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, EXPORT_SYMBOL(pcie_bandwidth_available); /** - * pcie_get_speed_cap - query for the PCI device's link speed capability + * pcie_get_supported_speeds - query Supported Link Speed Vector * @dev: PCI device to query * - * Query the PCI device speed capability. Return the maximum link speed - * supported by the device. + * Query @dev supported link speeds. + * + * Implementation Note in PCIe r6.0 sec 7.5.3.18 recommends determining + * supported link speeds using the Supported Link Speeds Vector in the Link + * Capabilities 2 Register (when available). + * + * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. + * + * Without Link Capabilities 2, i.e., prior to PCIe r3.0, Supported Link + * Speeds field in Link Capabilities is used and only 2.5 GT/s and 5.0 GT/s + * speeds were defined. + * + * For @dev without Supported Link Speed Vector, the field is synthesized + * from the Max Link Speed field in the Link Capabilities Register. + * + * Return: Supported Link Speeds Vector (+ reserved 0 at LSB). */ -enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) +u8 pcie_get_supported_speeds(struct pci_dev *dev) { u32 lnkcap2, lnkcap; + u8 speeds; /* - * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The - * implementation note there recommends using the Supported Link - * Speeds Vector in Link Capabilities 2 when supported. - * - * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software - * should use the Supported Link Speeds field in Link Capabilities, - * where only 2.5 GT/s and 5.0 GT/s speeds were defined. + * Speeds retain the reserved 0 at LSB before PCIe Supported Link + * Speeds Vector to allow using SLS Vector bit defines directly. */ pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2); + speeds = lnkcap2 & PCI_EXP_LNKCAP2_SLS; /* PCIe r3.0-compliant */ - if (lnkcap2) - return PCIE_LNKCAP2_SLS2SPEED(lnkcap2); + if (speeds) + return speeds; pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); + + /* Synthesize from the Max Link Speed field */ if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB) - return PCIE_SPEED_5_0GT; + speeds = PCI_EXP_LNKCAP2_SLS_5_0GB | PCI_EXP_LNKCAP2_SLS_2_5GB; else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB) - return PCIE_SPEED_2_5GT; + speeds = PCI_EXP_LNKCAP2_SLS_2_5GB; - return PCI_SPEED_UNKNOWN; + return speeds; +} + +/** + * pcie_get_speed_cap - query for the PCI device's link speed capability + * @dev: PCI device to query + * + * Query the PCI device speed capability. + * + * Return: the maximum link speed supported by the device. + */ +enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) +{ + return PCIE_LNKCAP2_SLS2SPEED(dev->supported_speeds); } EXPORT_SYMBOL(pcie_get_speed_cap); @@ -6653,8 +6686,7 @@ static void pci_request_resource_alignment(struct pci_dev *dev, int bar, } else { r->flags &= ~IORESOURCE_SIZEALIGN; r->flags |= IORESOURCE_STARTALIGN; - r->start = align; - r->end = r->start + size - 1; + resource_set_range(r, align, size); } r->flags |= IORESOURCE_UNSET; } @@ -6900,6 +6932,8 @@ static int __init pci_setup(char *str) pci_no_domains(); } else if (!strncmp(str, "noari", 5)) { pcie_ari_disabled = true; + } else if (!strncmp(str, "notph", 5)) { + pci_no_tph(); } else if (!strncmp(str, "cbiosize=", 9)) { pci_cardbus_io_size = memparse(str + 9, &str); } else if (!strncmp(str, "cbmemsize=", 10)) { diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 14d00ce45bfa..2e40fc63ba31 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -104,6 +104,7 @@ bool pci_reset_supported(struct pci_dev *dev); void pci_init_reset_methods(struct pci_dev *dev); int pci_bridge_secondary_bus_reset(struct pci_dev *dev); int pci_bus_error_reset(struct pci_dev *dev); +int __pci_reset_bus(struct pci_bus *bus); struct pci_cap_saved_data { u16 cap_nr; @@ -323,6 +324,9 @@ void __pci_bus_assign_resources(const struct pci_bus *bus, struct list_head *realloc_head, struct list_head *fail_head); bool pci_bus_clip_resource(struct pci_dev *dev, int idx); +void pci_walk_bus_locked(struct pci_bus *top, + int (*cb)(struct pci_dev *, void *), + void *userdata); const char *pci_resource_name(struct pci_dev *dev, unsigned int i); @@ -331,6 +335,17 @@ void pci_disable_bridge_window(struct pci_dev *dev); struct pci_bus *pci_bus_get(struct pci_bus *bus); void pci_bus_put(struct pci_bus *bus); +#define PCIE_LNKCAP_SLS2SPEED(lnkcap) \ +({ \ + ((lnkcap) == PCI_EXP_LNKCAP_SLS_64_0GB ? PCIE_SPEED_64_0GT : \ + (lnkcap) == PCI_EXP_LNKCAP_SLS_32_0GB ? PCIE_SPEED_32_0GT : \ + (lnkcap) == PCI_EXP_LNKCAP_SLS_16_0GB ? PCIE_SPEED_16_0GT : \ + (lnkcap) == PCI_EXP_LNKCAP_SLS_8_0GB ? PCIE_SPEED_8_0GT : \ + (lnkcap) == PCI_EXP_LNKCAP_SLS_5_0GB ? PCIE_SPEED_5_0GT : \ + (lnkcap) == PCI_EXP_LNKCAP_SLS_2_5GB ? PCIE_SPEED_2_5GT : \ + PCI_SPEED_UNKNOWN); \ +}) + /* PCIe link information from Link Capabilities 2 */ #define PCIE_LNKCAP2_SLS2SPEED(lnkcap2) \ ((lnkcap2) & PCI_EXP_LNKCAP2_SLS_64_0GB ? PCIE_SPEED_64_0GT : \ @@ -341,6 +356,15 @@ void pci_bus_put(struct pci_bus *bus); (lnkcap2) & PCI_EXP_LNKCAP2_SLS_2_5GB ? PCIE_SPEED_2_5GT : \ PCI_SPEED_UNKNOWN) +#define PCIE_LNKCTL2_TLS2SPEED(lnkctl2) \ + ((lnkctl2) == PCI_EXP_LNKCTL2_TLS_64_0GT ? PCIE_SPEED_64_0GT : \ + (lnkctl2) == PCI_EXP_LNKCTL2_TLS_32_0GT ? PCIE_SPEED_32_0GT : \ + (lnkctl2) == PCI_EXP_LNKCTL2_TLS_16_0GT ? PCIE_SPEED_16_0GT : \ + (lnkctl2) == PCI_EXP_LNKCTL2_TLS_8_0GT ? PCIE_SPEED_8_0GT : \ + (lnkctl2) == PCI_EXP_LNKCTL2_TLS_5_0GT ? PCIE_SPEED_5_0GT : \ + (lnkctl2) == PCI_EXP_LNKCTL2_TLS_2_5GT ? PCIE_SPEED_2_5GT : \ + PCI_SPEED_UNKNOWN) + /* PCIe speed to Mb/s reduced by encoding overhead */ #define PCIE_SPEED2MBS_ENC(speed) \ ((speed) == PCIE_SPEED_64_0GT ? 64000*1/1 : \ @@ -373,12 +397,16 @@ static inline int pcie_dev_speed_mbps(enum pci_bus_speed speed) return -EINVAL; } +u8 pcie_get_supported_speeds(struct pci_dev *dev); const char *pci_speed_string(enum pci_bus_speed speed); -enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev); -enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev); void __pcie_print_link_status(struct pci_dev *dev, bool verbose); void pcie_report_downtraining(struct pci_dev *dev); -void pcie_update_link_speed(struct pci_bus *bus, u16 link_status); + +static inline void __pcie_update_link_speed(struct pci_bus *bus, u16 linksta) +{ + bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS]; +} +void pcie_update_link_speed(struct pci_bus *bus); /* Single Root I/O Virtualization */ struct pci_sriov { @@ -469,10 +497,18 @@ static inline int pci_dev_set_disconnected(struct pci_dev *dev, void *unused) #define PCI_DEV_ADDED 0 #define PCI_DPC_RECOVERED 1 #define PCI_DPC_RECOVERING 2 +#define PCI_DEV_REMOVED 3 -static inline void pci_dev_assign_added(struct pci_dev *dev, bool added) +static inline void pci_dev_assign_added(struct pci_dev *dev) { - assign_bit(PCI_DEV_ADDED, &dev->priv_flags, added); + smp_mb__before_atomic(); + set_bit(PCI_DEV_ADDED, &dev->priv_flags); + smp_mb__after_atomic(); +} + +static inline bool pci_dev_test_and_clear_added(struct pci_dev *dev) +{ + return test_and_clear_bit(PCI_DEV_ADDED, &dev->priv_flags); } static inline bool pci_dev_is_added(const struct pci_dev *dev) @@ -480,6 +516,11 @@ static inline bool pci_dev_is_added(const struct pci_dev *dev) return test_bit(PCI_DEV_ADDED, &dev->priv_flags); } +static inline bool pci_dev_test_and_set_removed(struct pci_dev *dev) +{ + return test_and_set_bit(PCI_DEV_REMOVED, &dev->priv_flags); +} + #ifdef CONFIG_PCIEAER #include <linux/aer.h> @@ -597,6 +638,18 @@ static inline int pci_iov_bus_range(struct pci_bus *bus) #endif /* CONFIG_PCI_IOV */ +#ifdef CONFIG_PCIE_TPH +void pci_restore_tph_state(struct pci_dev *dev); +void pci_save_tph_state(struct pci_dev *dev); +void pci_no_tph(void); +void pci_tph_init(struct pci_dev *dev); +#else +static inline void pci_restore_tph_state(struct pci_dev *dev) { } +static inline void pci_save_tph_state(struct pci_dev *dev) { } +static inline void pci_no_tph(void) { } +static inline void pci_tph_init(struct pci_dev *dev) { } +#endif + #ifdef CONFIG_PCIE_PTM void pci_ptm_init(struct pci_dev *dev); void pci_save_ptm_state(struct pci_dev *dev); @@ -692,6 +745,17 @@ static inline void pcie_set_ecrc_checking(struct pci_dev *dev) { } static inline void pcie_ecrc_get_policy(char *str) { } #endif +#ifdef CONFIG_PCIEPORTBUS +void pcie_reset_lbms_count(struct pci_dev *port); +int pcie_lbms_count(struct pci_dev *port, unsigned long *val); +#else +static inline void pcie_reset_lbms_count(struct pci_dev *port) {} +static inline int pcie_lbms_count(struct pci_dev *port, unsigned long *val) +{ + return -EOPNOTSUPP; +} +#endif + struct pci_dev_reset_methods { u16 vendor; u16 device; @@ -746,6 +810,7 @@ void pci_set_bus_of_node(struct pci_bus *bus); void pci_release_bus_of_node(struct pci_bus *bus); int devm_of_pci_bridge_init(struct device *dev, struct pci_host_bridge *bridge); +bool of_pci_supply_present(struct device_node *np); #else static inline int @@ -793,6 +858,10 @@ static inline int devm_of_pci_bridge_init(struct device *dev, struct pci_host_br return 0; } +static inline bool of_pci_supply_present(struct device_node *np) +{ + return false; +} #endif /* CONFIG_OF */ struct of_changeset; diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index 6461aa93fe76..53ccab62314d 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -4,7 +4,7 @@ pcieportdrv-y := portdrv.o rcec.o -obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o +obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o bwctrl.o obj-y += aspm.o obj-$(CONFIG_PCIEAER) += aer.o err.o diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 13b8586924ea..80c5ba8d8296 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -180,7 +180,8 @@ static int disable_ecrc_checking(struct pci_dev *dev) } /** - * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based on global policy + * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based + * on global policy * @dev: the PCI device */ void pcie_set_ecrc_checking(struct pci_dev *dev) @@ -1148,14 +1149,16 @@ static void aer_recover_work_func(struct work_struct *work) continue; } pci_print_aer(pdev, entry.severity, entry.regs); + /* - * Memory for aer_capability_regs(entry.regs) is being allocated from the - * ghes_estatus_pool to protect it from overwriting when multiple sections - * are present in the error status. Thus free the same after processing - * the data. + * Memory for aer_capability_regs(entry.regs) is being + * allocated from the ghes_estatus_pool to protect it from + * overwriting when multiple sections are present in the + * error status. Thus free the same after processing the + * data. */ ghes_estatus_pool_region_free((unsigned long)entry.regs, - sizeof(struct aer_capability_regs)); + sizeof(struct aer_capability_regs)); if (entry.severity == AER_NONFATAL) pcie_do_recovery(pdev, pci_channel_io_normal, diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index cee2365e54b8..28567d457613 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -805,6 +805,15 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl); pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl); + /* Disable L0s/L1 before updating L1SS config */ + if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || + FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { + pcie_capability_write_word(child, PCI_EXP_LNKCTL, + child_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, + parent_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); + } + /* * Setup L0s state * @@ -829,6 +838,13 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) aspm_l1ss_init(link); + /* Restore L0s/L1 if they were enabled */ + if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || + FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_lnkctl); + pcie_capability_write_word(child, PCI_EXP_LNKCTL, child_lnkctl); + } + /* Save default state */ link->aspm_default = link->aspm_enabled; @@ -845,25 +861,28 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) } } -/* Configure the ASPM L1 substates */ +/* Configure the ASPM L1 substates. Caller must disable L1 first. */ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) { - u32 val, enable_req; + u32 val; struct pci_dev *child = link->downstream, *parent = link->pdev; - enable_req = (link->aspm_enabled ^ state) & state; + val = 0; + if (state & PCIE_LINK_STATE_L1_1) + val |= PCI_L1SS_CTL1_ASPM_L1_1; + if (state & PCIE_LINK_STATE_L1_2) + val |= PCI_L1SS_CTL1_ASPM_L1_2; + if (state & PCIE_LINK_STATE_L1_1_PCIPM) + val |= PCI_L1SS_CTL1_PCIPM_L1_1; + if (state & PCIE_LINK_STATE_L1_2_PCIPM) + val |= PCI_L1SS_CTL1_PCIPM_L1_2; /* - * Here are the rules specified in the PCIe spec for enabling L1SS: - * - When enabling L1.x, enable bit at parent first, then at child - * - When disabling L1.x, disable bit at child first, then at parent - * - When enabling ASPM L1.x, need to disable L1 - * (at child followed by parent). - * - The ASPM/PCIPM L1.2 must be disabled while programming timing + * PCIe r6.2, sec 5.5.4, rules for enabling L1 PM Substates: + * - Clear L1.x enable bits at child first, then at parent + * - Set L1.x enable bits at parent first, then at child + * - ASPM/PCIPM L1.2 must be disabled while programming timing * parameters - * - * To keep it simple, disable all L1SS bits first, and later enable - * what is needed. */ /* Disable all L1 substates */ @@ -871,26 +890,6 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) PCI_L1SS_CTL1_L1SS_MASK, 0); pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_L1SS_MASK, 0); - /* - * If needed, disable L1, and it gets enabled later - * in pcie_config_aspm_link(). - */ - if (enable_req & (PCIE_LINK_STATE_L1_1 | PCIE_LINK_STATE_L1_2)) { - pcie_capability_clear_word(child, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPM_L1); - pcie_capability_clear_word(parent, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPM_L1); - } - - val = 0; - if (state & PCIE_LINK_STATE_L1_1) - val |= PCI_L1SS_CTL1_ASPM_L1_1; - if (state & PCIE_LINK_STATE_L1_2) - val |= PCI_L1SS_CTL1_ASPM_L1_2; - if (state & PCIE_LINK_STATE_L1_1_PCIPM) - val |= PCI_L1SS_CTL1_PCIPM_L1_1; - if (state & PCIE_LINK_STATE_L1_2_PCIPM) - val |= PCI_L1SS_CTL1_PCIPM_L1_2; /* Enable what we need to enable */ pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, @@ -937,21 +936,30 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) dwstream |= PCI_EXP_LNKCTL_ASPM_L1; } + /* + * Per PCIe r6.2, sec 5.5.4, setting either or both of the enable + * bits for ASPM L1 PM Substates must be done while ASPM L1 is + * disabled. Disable L1 here and apply new configuration after L1SS + * configuration has been completed. + * + * Per sec 7.5.3.7, when disabling ASPM L1, software must disable + * it in the Downstream component prior to disabling it in the + * Upstream component, and ASPM L1 must be enabled in the Upstream + * component prior to enabling it in the Downstream component. + * + * Sec 7.5.3.7 also recommends programming the same ASPM Control + * value for all functions of a multi-function device. + */ + list_for_each_entry(child, &linkbus->devices, bus_list) + pcie_config_aspm_dev(child, 0); + pcie_config_aspm_dev(parent, 0); + if (link->aspm_capable & PCIE_LINK_STATE_L1SS) pcie_config_aspm_l1ss(link, state); - /* - * Spec 2.0 suggests all functions should be configured the - * same setting for ASPM. Enabling ASPM L1 should be done in - * upstream component first and then downstream, and vice - * versa for disabling ASPM L1. Spec doesn't mention L0S. - */ - if (state & PCIE_LINK_STATE_L1) - pcie_config_aspm_dev(parent, upstream); + pcie_config_aspm_dev(parent, upstream); list_for_each_entry(child, &linkbus->devices, bus_list) pcie_config_aspm_dev(child, dwstream); - if (!(state & PCIE_LINK_STATE_L1)) - pcie_config_aspm_dev(parent, upstream); link->aspm_enabled = state; @@ -1442,6 +1450,9 @@ static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked) * touch the LNKCTL register. Also note that this does not enable states * disabled by pci_disable_link_state(). Return 0 or a negative errno. * + * Note: Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per + * PCIe r6.0, sec 5.5.4. + * * @pdev: PCI device * @state: Mask of ASPM link states to enable */ @@ -1458,6 +1469,9 @@ EXPORT_SYMBOL(pci_enable_link_state); * can't touch the LNKCTL register. Also note that this does not enable states * disabled by pci_disable_link_state(). Return 0 or a negative errno. * + * Note: Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per + * PCIe r6.0, sec 5.5.4. + * * @pdev: PCI device * @state: Mask of ASPM link states to enable * diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c new file mode 100644 index 000000000000..b59cacc740fa --- /dev/null +++ b/drivers/pci/pcie/bwctrl.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe bandwidth controller + * + * Author: Alexandru Gagniuc <mr.nuke.me@gmail.com> + * + * Copyright (C) 2019 Dell Inc + * Copyright (C) 2023-2024 Intel Corporation + * + * The PCIe bandwidth controller provides a way to alter PCIe Link Speeds + * and notify the operating system when the Link Width or Speed changes. The + * notification capability is required for all Root Ports and Downstream + * Ports supporting Link Width wider than x1 and/or multiple Link Speeds. + * + * This service port driver hooks into the Bandwidth Notification interrupt + * watching for changes or links becoming degraded in operation. It updates + * the cached Current Link Speed that is exposed to user space through sysfs. + */ + +#define dev_fmt(fmt) "bwctrl: " fmt + +#include <linux/atomic.h> +#include <linux/bitops.h> +#include <linux/bits.h> +#include <linux/cleanup.h> +#include <linux/errno.h> +#include <linux/interrupt.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci-bwctrl.h> +#include <linux/rwsem.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "../pci.h" +#include "portdrv.h" + +/** + * struct pcie_bwctrl_data - PCIe bandwidth controller + * @set_speed_mutex: Serializes link speed changes + * @lbms_count: Count for LBMS (since last reset) + * @cdev: Thermal cooling device associated with the port + */ +struct pcie_bwctrl_data { + struct mutex set_speed_mutex; + atomic_t lbms_count; + struct thermal_cooling_device *cdev; +}; + +/* + * Prevent port removal during LBMS count accessors and Link Speed changes. + * + * These have to be differentiated because pcie_bwctrl_change_speed() calls + * pcie_retrain_link() which uses LBMS count reset accessor on success + * (using just one rwsem triggers "possible recursive locking detected" + * warning). + */ +static DECLARE_RWSEM(pcie_bwctrl_lbms_rwsem); +static DECLARE_RWSEM(pcie_bwctrl_setspeed_rwsem); + +static bool pcie_valid_speed(enum pci_bus_speed speed) +{ + return (speed >= PCIE_SPEED_2_5GT) && (speed <= PCIE_SPEED_64_0GT); +} + +static u16 pci_bus_speed2lnkctl2(enum pci_bus_speed speed) +{ + static const u8 speed_conv[] = { + [PCIE_SPEED_2_5GT] = PCI_EXP_LNKCTL2_TLS_2_5GT, + [PCIE_SPEED_5_0GT] = PCI_EXP_LNKCTL2_TLS_5_0GT, + [PCIE_SPEED_8_0GT] = PCI_EXP_LNKCTL2_TLS_8_0GT, + [PCIE_SPEED_16_0GT] = PCI_EXP_LNKCTL2_TLS_16_0GT, + [PCIE_SPEED_32_0GT] = PCI_EXP_LNKCTL2_TLS_32_0GT, + [PCIE_SPEED_64_0GT] = PCI_EXP_LNKCTL2_TLS_64_0GT, + }; + + if (WARN_ON_ONCE(!pcie_valid_speed(speed))) + return 0; + + return speed_conv[speed]; +} + +static inline u16 pcie_supported_speeds2target_speed(u8 supported_speeds) +{ + return __fls(supported_speeds); +} + +/** + * pcie_bwctrl_select_speed - Select Target Link Speed + * @port: PCIe Port + * @speed_req: Requested PCIe Link Speed + * + * Select Target Link Speed by take into account Supported Link Speeds of + * both the Root Port and the Endpoint. + * + * Return: Target Link Speed (1=2.5GT/s, 2=5GT/s, 3=8GT/s, etc.) + */ +static u16 pcie_bwctrl_select_speed(struct pci_dev *port, enum pci_bus_speed speed_req) +{ + struct pci_bus *bus = port->subordinate; + u8 desired_speeds, supported_speeds; + struct pci_dev *dev; + + desired_speeds = GENMASK(pci_bus_speed2lnkctl2(speed_req), + __fls(PCI_EXP_LNKCAP2_SLS_2_5GB)); + + supported_speeds = port->supported_speeds; + if (bus) { + down_read(&pci_bus_sem); + dev = list_first_entry_or_null(&bus->devices, struct pci_dev, bus_list); + if (dev) + supported_speeds &= dev->supported_speeds; + up_read(&pci_bus_sem); + } + if (!supported_speeds) + return PCI_EXP_LNKCAP2_SLS_2_5GB; + + return pcie_supported_speeds2target_speed(supported_speeds & desired_speeds); +} + +static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool use_lt) +{ + int ret; + + ret = pcie_capability_clear_and_set_word(port, PCI_EXP_LNKCTL2, + PCI_EXP_LNKCTL2_TLS, target_speed); + if (ret != PCIBIOS_SUCCESSFUL) + return pcibios_err_to_errno(ret); + + ret = pcie_retrain_link(port, use_lt); + if (ret < 0) + return ret; + + /* + * Ensure link speed updates also with platforms that have problems + * with notifications. + */ + if (port->subordinate) + pcie_update_link_speed(port->subordinate); + + return 0; +} + +/** + * pcie_set_target_speed - Set downstream Link Speed for PCIe Port + * @port: PCIe Port + * @speed_req: Requested PCIe Link Speed + * @use_lt: Wait for the LT or DLLLA bit to detect the end of link training + * + * Attempt to set PCIe Port Link Speed to @speed_req. @speed_req may be + * adjusted downwards to the best speed supported by both the Port and PCIe + * Device underneath it. + * + * Return: + * * 0 - on success + * * -EINVAL - @speed_req is not a PCIe Link Speed + * * -ENODEV - @port is not controllable + * * -ETIMEDOUT - changing Link Speed took too long + * * -EAGAIN - Link Speed was changed but @speed_req was not achieved + */ +int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, + bool use_lt) +{ + struct pci_bus *bus = port->subordinate; + u16 target_speed; + int ret; + + if (WARN_ON_ONCE(!pcie_valid_speed(speed_req))) + return -EINVAL; + + if (bus && bus->cur_bus_speed == speed_req) + return 0; + + target_speed = pcie_bwctrl_select_speed(port, speed_req); + + scoped_guard(rwsem_read, &pcie_bwctrl_setspeed_rwsem) { + struct pcie_bwctrl_data *data = port->link_bwctrl; + + /* + * port->link_bwctrl is NULL during initial scan when called + * e.g. from the Target Speed quirk. + */ + if (data) + mutex_lock(&data->set_speed_mutex); + + ret = pcie_bwctrl_change_speed(port, target_speed, use_lt); + + if (data) + mutex_unlock(&data->set_speed_mutex); + } + + /* + * Despite setting higher speed into the Target Link Speed, empty + * bus won't train to 5GT+ speeds. + */ + if (!ret && bus && bus->cur_bus_speed != speed_req && + !list_empty(&bus->devices)) + ret = -EAGAIN; + + return ret; +} + +static void pcie_bwnotif_enable(struct pcie_device *srv) +{ + struct pcie_bwctrl_data *data = srv->port->link_bwctrl; + struct pci_dev *port = srv->port; + u16 link_status; + int ret; + + /* Count LBMS seen so far as one */ + ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status); + if (ret == PCIBIOS_SUCCESSFUL && link_status & PCI_EXP_LNKSTA_LBMS) + atomic_inc(&data->lbms_count); + + pcie_capability_set_word(port, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); + pcie_capability_write_word(port, PCI_EXP_LNKSTA, + PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS); + + /* + * Update after enabling notifications & clearing status bits ensures + * link speed is up to date. + */ + pcie_update_link_speed(port->subordinate); +} + +static void pcie_bwnotif_disable(struct pci_dev *port) +{ + pcie_capability_clear_word(port, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); +} + +static irqreturn_t pcie_bwnotif_irq(int irq, void *context) +{ + struct pcie_device *srv = context; + struct pcie_bwctrl_data *data = srv->port->link_bwctrl; + struct pci_dev *port = srv->port; + u16 link_status, events; + int ret; + + ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status); + if (ret != PCIBIOS_SUCCESSFUL) + return IRQ_NONE; + + events = link_status & (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS); + if (!events) + return IRQ_NONE; + + if (events & PCI_EXP_LNKSTA_LBMS) + atomic_inc(&data->lbms_count); + + pcie_capability_write_word(port, PCI_EXP_LNKSTA, events); + + /* + * Interrupts will not be triggered from any further Link Speed + * change until LBMS is cleared by the write. Therefore, re-read the + * speed (inside pcie_update_link_speed()) after LBMS has been + * cleared to avoid missing link speed changes. + */ + pcie_update_link_speed(port->subordinate); + + return IRQ_HANDLED; +} + +void pcie_reset_lbms_count(struct pci_dev *port) +{ + struct pcie_bwctrl_data *data; + + guard(rwsem_read)(&pcie_bwctrl_lbms_rwsem); + data = port->link_bwctrl; + if (data) + atomic_set(&data->lbms_count, 0); + else + pcie_capability_write_word(port, PCI_EXP_LNKSTA, + PCI_EXP_LNKSTA_LBMS); +} + +int pcie_lbms_count(struct pci_dev *port, unsigned long *val) +{ + struct pcie_bwctrl_data *data; + + guard(rwsem_read)(&pcie_bwctrl_lbms_rwsem); + data = port->link_bwctrl; + if (!data) + return -ENOTTY; + + *val = atomic_read(&data->lbms_count); + + return 0; +} + +static int pcie_bwnotif_probe(struct pcie_device *srv) +{ + struct pci_dev *port = srv->port; + int ret; + + struct pcie_bwctrl_data *data = devm_kzalloc(&srv->device, + sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + ret = devm_mutex_init(&srv->device, &data->set_speed_mutex); + if (ret) + return ret; + + ret = devm_request_irq(&srv->device, srv->irq, pcie_bwnotif_irq, + IRQF_SHARED, "PCIe bwctrl", srv); + if (ret) + return ret; + + scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { + scoped_guard(rwsem_write, &pcie_bwctrl_lbms_rwsem) { + port->link_bwctrl = no_free_ptr(data); + pcie_bwnotif_enable(srv); + } + } + + pci_dbg(port, "enabled with IRQ %d\n", srv->irq); + + /* Don't fail on errors. Don't leave IS_ERR() "pointer" into ->cdev */ + port->link_bwctrl->cdev = pcie_cooling_device_register(port); + if (IS_ERR(port->link_bwctrl->cdev)) + port->link_bwctrl->cdev = NULL; + + return 0; +} + +static void pcie_bwnotif_remove(struct pcie_device *srv) +{ + struct pcie_bwctrl_data *data = srv->port->link_bwctrl; + + pcie_cooling_device_unregister(data->cdev); + + pcie_bwnotif_disable(srv->port); + + scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) + scoped_guard(rwsem_write, &pcie_bwctrl_lbms_rwsem) + srv->port->link_bwctrl = NULL; +} + +static int pcie_bwnotif_suspend(struct pcie_device *srv) +{ + pcie_bwnotif_disable(srv->port); + return 0; +} + +static int pcie_bwnotif_resume(struct pcie_device *srv) +{ + pcie_bwnotif_enable(srv); + return 0; +} + +static struct pcie_port_service_driver pcie_bwctrl_driver = { + .name = "pcie_bwctrl", + .port_type = PCIE_ANY_PORT, + .service = PCIE_PORT_SERVICE_BWCTRL, + .probe = pcie_bwnotif_probe, + .suspend = pcie_bwnotif_suspend, + .resume = pcie_bwnotif_resume, + .remove = pcie_bwnotif_remove, +}; + +int __init pcie_bwctrl_init(void) +{ + return pcie_port_service_register(&pcie_bwctrl_driver); +} diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 6af5e0425872..5e10306b6308 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -68,7 +68,7 @@ static int pcie_message_numbers(struct pci_dev *dev, int mask, */ if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP | - PCIE_PORT_SERVICE_BWNOTIF)) { + PCIE_PORT_SERVICE_BWCTRL)) { pcie_capability_read_word(dev, PCI_EXP_FLAGS, ®16); *pme = FIELD_GET(PCI_EXP_FLAGS_IRQ, reg16); nvec = *pme + 1; @@ -150,11 +150,11 @@ static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask) /* PME, hotplug and bandwidth notification share an MSI/MSI-X vector */ if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP | - PCIE_PORT_SERVICE_BWNOTIF)) { + PCIE_PORT_SERVICE_BWCTRL)) { pcie_irq = pci_irq_vector(dev, pme); irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pcie_irq; irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pcie_irq; - irqs[PCIE_PORT_SERVICE_BWNOTIF_SHIFT] = pcie_irq; + irqs[PCIE_PORT_SERVICE_BWCTRL_SHIFT] = pcie_irq; } if (mask & PCIE_PORT_SERVICE_AER) @@ -271,7 +271,7 @@ static int get_port_device_capability(struct pci_dev *dev) pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &linkcap); if (linkcap & PCI_EXP_LNKCAP_LBNC) - services |= PCIE_PORT_SERVICE_BWNOTIF; + services |= PCIE_PORT_SERVICE_BWCTRL; } return services; @@ -828,6 +828,7 @@ static void __init pcie_init_services(void) pcie_aer_init(); pcie_pme_init(); pcie_dpc_init(); + pcie_bwctrl_init(); pcie_hp_init(); } diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index 12c89ea0313b..bd29d1cc7b8b 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -20,8 +20,8 @@ #define PCIE_PORT_SERVICE_HP (1 << PCIE_PORT_SERVICE_HP_SHIFT) #define PCIE_PORT_SERVICE_DPC_SHIFT 3 /* Downstream Port Containment */ #define PCIE_PORT_SERVICE_DPC (1 << PCIE_PORT_SERVICE_DPC_SHIFT) -#define PCIE_PORT_SERVICE_BWNOTIF_SHIFT 4 /* Bandwidth notification */ -#define PCIE_PORT_SERVICE_BWNOTIF (1 << PCIE_PORT_SERVICE_BWNOTIF_SHIFT) +#define PCIE_PORT_SERVICE_BWCTRL_SHIFT 4 /* Bandwidth Controller (notifications) */ +#define PCIE_PORT_SERVICE_BWCTRL (1 << PCIE_PORT_SERVICE_BWCTRL_SHIFT) #define PCIE_PORT_DEVICE_MAXSERVICES 5 @@ -51,6 +51,8 @@ int pcie_dpc_init(void); static inline int pcie_dpc_init(void) { return 0; } #endif +int pcie_bwctrl_init(void); + /* Port Type */ #define PCIE_ANY_PORT (~0) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index f1615805f5b0..2e81ab0f5a25 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -543,15 +543,15 @@ void pci_read_bridge_bases(struct pci_bus *child) pci_read_bridge_mmio(child->self, child->resource[1], false); pci_read_bridge_mmio_pref(child->self, child->resource[2], false); - if (dev->transparent) { - pci_bus_for_each_resource(child->parent, res) { - if (res && res->flags) { - pci_bus_add_resource(child, res, - PCI_SUBTRACTIVE_DECODE); - pci_info(dev, " bridge window %pR (subtractive decode)\n", - res); - } - } + if (!dev->transparent) + return; + + pci_bus_for_each_resource(child->parent, res) { + if (!res || !res->flags) + continue; + + pci_bus_add_resource(child, res); + pci_info(dev, " bridge window %pR (subtractive decode)\n", res); } } @@ -742,9 +742,13 @@ const char *pci_speed_string(enum pci_bus_speed speed) } EXPORT_SYMBOL_GPL(pci_speed_string); -void pcie_update_link_speed(struct pci_bus *bus, u16 linksta) +void pcie_update_link_speed(struct pci_bus *bus) { - bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS]; + struct pci_dev *bridge = bus->self; + u16 linksta; + + pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta); + __pcie_update_link_speed(bus, linksta); } EXPORT_SYMBOL_GPL(pcie_update_link_speed); @@ -827,13 +831,11 @@ static void pci_set_bus_speed(struct pci_bus *bus) if (pci_is_pcie(bridge)) { u32 linkcap; - u16 linksta; pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &linkcap); bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS]; - pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta); - pcie_update_link_speed(bus, linksta); + pcie_update_link_speed(bus); } } @@ -1032,7 +1034,7 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (res->flags & IORESOURCE_BUS) pci_bus_insert_busn_res(bus, bus->number, res->end); else - pci_bus_add_resource(bus, res, 0); + pci_bus_add_resource(bus, res); if (offset) { if (resource_type(res) == IORESOURCE_IO) @@ -1633,23 +1635,33 @@ static void set_pcie_thunderbolt(struct pci_dev *dev) static void set_pcie_untrusted(struct pci_dev *dev) { - struct pci_dev *parent; + struct pci_dev *parent = pci_upstream_bridge(dev); + if (!parent) + return; /* - * If the upstream bridge is untrusted we treat this device + * If the upstream bridge is untrusted we treat this device as * untrusted as well. */ - parent = pci_upstream_bridge(dev); - if (parent && (parent->untrusted || parent->external_facing)) + if (parent->untrusted) { + dev->untrusted = true; + return; + } + + if (arch_pci_dev_is_removable(dev)) { + pci_dbg(dev, "marking as untrusted\n"); dev->untrusted = true; + } } static void pci_set_removable(struct pci_dev *dev) { struct pci_dev *parent = pci_upstream_bridge(dev); + if (!parent) + return; /* - * We (only) consider everything downstream from an external_facing + * We (only) consider everything tunneled below an external_facing * device to be removable by the user. We're mainly concerned with * consumer platforms with user accessible thunderbolt ports that are * vulnerable to DMA attacks, and we expect those ports to be marked by @@ -1659,9 +1671,15 @@ static void pci_set_removable(struct pci_dev *dev) * accessible to user / may not be removed by end user, and thus not * exposed as "removable" to userspace. */ - if (parent && - (parent->external_facing || dev_is_removable(&parent->dev))) + if (dev_is_removable(&parent->dev)) { + dev_set_removable(&dev->dev, DEVICE_REMOVABLE); + return; + } + + if (arch_pci_dev_is_removable(dev)) { + pci_dbg(dev, "marking as removable\n"); dev_set_removable(&dev->dev, DEVICE_REMOVABLE); + } } /** @@ -1947,6 +1965,9 @@ int pci_setup_device(struct pci_dev *dev) set_pcie_untrusted(dev); + if (pci_is_pcie(dev)) + dev->supported_speeds = pcie_get_supported_speeds(dev); + /* "Unknown power state" */ dev->current_state = PCI_UNKNOWN; @@ -2495,6 +2516,7 @@ static void pci_init_capabilities(struct pci_dev *dev) pci_dpc_init(dev); /* Downstream Port Containment */ pci_rcec_init(dev); /* Root Complex Event Collector */ pci_doe_init(dev); /* Data Object Exchange */ + pci_tph_init(dev); /* TLP Processing Hints */ pcie_report_downtraining(dev); pci_init_reset_methods(dev); @@ -3108,6 +3130,17 @@ int pci_host_probe(struct pci_host_bridge *bridge) pci_lock_rescan_remove(); pci_bus_add_devices(bus); pci_unlock_rescan_remove(); + + /* + * Ensure pm_runtime_enable() is called for the controller drivers + * before calling pci_host_probe(). The PM framework expects that + * if the parent device supports runtime PM, it will be enabled + * before child runtime PM is enabled. + */ + pm_runtime_set_active(&bridge->dev); + pm_runtime_no_callbacks(&bridge->dev); + devm_pm_runtime_enable(&bridge->dev); + return 0; } EXPORT_SYMBOL_GPL(pci_host_probe); diff --git a/drivers/pci/pwrctl/Makefile b/drivers/pci/pwrctl/Makefile deleted file mode 100644 index d308aae4800c..000000000000 --- a/drivers/pci/pwrctl/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -obj-$(CONFIG_PCI_PWRCTL) += pci-pwrctl-core.o -pci-pwrctl-core-y := core.o - -obj-$(CONFIG_PCI_PWRCTL_PWRSEQ) += pci-pwrctl-pwrseq.o diff --git a/drivers/pci/pwrctl/core.c b/drivers/pci/pwrctl/core.c deleted file mode 100644 index 01d913b60316..000000000000 --- a/drivers/pci/pwrctl/core.c +++ /dev/null @@ -1,157 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2024 Linaro Ltd. - */ - -#include <linux/device.h> -#include <linux/export.h> -#include <linux/kernel.h> -#include <linux/pci.h> -#include <linux/pci-pwrctl.h> -#include <linux/property.h> -#include <linux/slab.h> - -static int pci_pwrctl_notify(struct notifier_block *nb, unsigned long action, - void *data) -{ - struct pci_pwrctl *pwrctl = container_of(nb, struct pci_pwrctl, nb); - struct device *dev = data; - - if (dev_fwnode(dev) != dev_fwnode(pwrctl->dev)) - return NOTIFY_DONE; - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: - /* - * We will have two struct device objects bound to two different - * drivers on different buses but consuming the same DT node. We - * must not bind the pins twice in this case but only once for - * the first device to be added. - * - * If we got here then the PCI device is the second after the - * power control platform device. Mark its OF node as reused. - */ - dev->of_node_reused = true; - break; - case BUS_NOTIFY_BOUND_DRIVER: - pwrctl->link = device_link_add(dev, pwrctl->dev, - DL_FLAG_AUTOREMOVE_CONSUMER); - if (!pwrctl->link) - dev_err(pwrctl->dev, "Failed to add device link\n"); - break; - case BUS_NOTIFY_UNBOUND_DRIVER: - if (pwrctl->link) - device_link_remove(dev, pwrctl->dev); - break; - } - - return NOTIFY_DONE; -} - -static void rescan_work_func(struct work_struct *work) -{ - struct pci_pwrctl *pwrctl = container_of(work, struct pci_pwrctl, work); - - pci_lock_rescan_remove(); - pci_rescan_bus(to_pci_dev(pwrctl->dev->parent)->bus); - pci_unlock_rescan_remove(); -} - -/** - * pci_pwrctl_init() - Initialize the PCI power control context struct - * - * @pwrctl: PCI power control data - * @dev: Parent device - */ -void pci_pwrctl_init(struct pci_pwrctl *pwrctl, struct device *dev) -{ - pwrctl->dev = dev; - INIT_WORK(&pwrctl->work, rescan_work_func); -} -EXPORT_SYMBOL_GPL(pci_pwrctl_init); - -/** - * pci_pwrctl_device_set_ready() - Notify the pwrctl subsystem that the PCI - * device is powered-up and ready to be detected. - * - * @pwrctl: PCI power control data. - * - * Returns: - * 0 on success, negative error number on error. - * - * Note: - * This function returning 0 doesn't mean the device was detected. It means, - * that the bus rescan was successfully started. The device will get bound to - * its PCI driver asynchronously. - */ -int pci_pwrctl_device_set_ready(struct pci_pwrctl *pwrctl) -{ - int ret; - - if (!pwrctl->dev) - return -ENODEV; - - pwrctl->nb.notifier_call = pci_pwrctl_notify; - ret = bus_register_notifier(&pci_bus_type, &pwrctl->nb); - if (ret) - return ret; - - schedule_work(&pwrctl->work); - - return 0; -} -EXPORT_SYMBOL_GPL(pci_pwrctl_device_set_ready); - -/** - * pci_pwrctl_device_unset_ready() - Notify the pwrctl subsystem that the PCI - * device is about to be powered-down. - * - * @pwrctl: PCI power control data. - */ -void pci_pwrctl_device_unset_ready(struct pci_pwrctl *pwrctl) -{ - /* - * We don't have to delete the link here. Typically, this function - * is only called when the power control device is being detached. If - * it is being detached then the child PCI device must have already - * been unbound too or the device core wouldn't let us unbind. - */ - bus_unregister_notifier(&pci_bus_type, &pwrctl->nb); -} -EXPORT_SYMBOL_GPL(pci_pwrctl_device_unset_ready); - -static void devm_pci_pwrctl_device_unset_ready(void *data) -{ - struct pci_pwrctl *pwrctl = data; - - pci_pwrctl_device_unset_ready(pwrctl); -} - -/** - * devm_pci_pwrctl_device_set_ready - Managed variant of - * pci_pwrctl_device_set_ready(). - * - * @dev: Device managing this pwrctl provider. - * @pwrctl: PCI power control data. - * - * Returns: - * 0 on success, negative error number on error. - */ -int devm_pci_pwrctl_device_set_ready(struct device *dev, - struct pci_pwrctl *pwrctl) -{ - int ret; - - ret = pci_pwrctl_device_set_ready(pwrctl); - if (ret) - return ret; - - return devm_add_action_or_reset(dev, - devm_pci_pwrctl_device_unset_ready, - pwrctl); -} -EXPORT_SYMBOL_GPL(devm_pci_pwrctl_device_set_ready); - -MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@linaro.org>"); -MODULE_DESCRIPTION("PCI Device Power Control core driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/pci/pwrctl/Kconfig b/drivers/pci/pwrctrl/Kconfig index 54589bb2403b..54589bb2403b 100644 --- a/drivers/pci/pwrctl/Kconfig +++ b/drivers/pci/pwrctrl/Kconfig diff --git a/drivers/pci/pwrctrl/Makefile b/drivers/pci/pwrctrl/Makefile new file mode 100644 index 000000000000..75c7ce531c7e --- /dev/null +++ b/drivers/pci/pwrctrl/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_PCI_PWRCTL) += pci-pwrctrl-core.o +pci-pwrctrl-core-y := core.o + +obj-$(CONFIG_PCI_PWRCTL_PWRSEQ) += pci-pwrctrl-pwrseq.o diff --git a/drivers/pci/pwrctrl/core.c b/drivers/pci/pwrctrl/core.c new file mode 100644 index 000000000000..2fb174db91e5 --- /dev/null +++ b/drivers/pci/pwrctrl/core.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Linaro Ltd. + */ + +#include <linux/device.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci-pwrctrl.h> +#include <linux/property.h> +#include <linux/slab.h> + +static int pci_pwrctrl_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_pwrctrl *pwrctrl = container_of(nb, struct pci_pwrctrl, nb); + struct device *dev = data; + + if (dev_fwnode(dev) != dev_fwnode(pwrctrl->dev)) + return NOTIFY_DONE; + + switch (action) { + case BUS_NOTIFY_ADD_DEVICE: + /* + * We will have two struct device objects bound to two different + * drivers on different buses but consuming the same DT node. We + * must not bind the pins twice in this case but only once for + * the first device to be added. + * + * If we got here then the PCI device is the second after the + * power control platform device. Mark its OF node as reused. + */ + dev->of_node_reused = true; + break; + } + + return NOTIFY_DONE; +} + +static void rescan_work_func(struct work_struct *work) +{ + struct pci_pwrctrl *pwrctrl = container_of(work, + struct pci_pwrctrl, work); + + pci_lock_rescan_remove(); + pci_rescan_bus(to_pci_dev(pwrctrl->dev->parent)->bus); + pci_unlock_rescan_remove(); +} + +/** + * pci_pwrctrl_init() - Initialize the PCI power control context struct + * + * @pwrctrl: PCI power control data + * @dev: Parent device + */ +void pci_pwrctrl_init(struct pci_pwrctrl *pwrctrl, struct device *dev) +{ + pwrctrl->dev = dev; + INIT_WORK(&pwrctrl->work, rescan_work_func); +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_init); + +/** + * pci_pwrctrl_device_set_ready() - Notify the pwrctrl subsystem that the PCI + * device is powered-up and ready to be detected. + * + * @pwrctrl: PCI power control data. + * + * Returns: + * 0 on success, negative error number on error. + * + * Note: + * This function returning 0 doesn't mean the device was detected. It means, + * that the bus rescan was successfully started. The device will get bound to + * its PCI driver asynchronously. + */ +int pci_pwrctrl_device_set_ready(struct pci_pwrctrl *pwrctrl) +{ + int ret; + + if (!pwrctrl->dev) + return -ENODEV; + + pwrctrl->nb.notifier_call = pci_pwrctrl_notify; + ret = bus_register_notifier(&pci_bus_type, &pwrctrl->nb); + if (ret) + return ret; + + schedule_work(&pwrctrl->work); + + return 0; +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_device_set_ready); + +/** + * pci_pwrctrl_device_unset_ready() - Notify the pwrctrl subsystem that the PCI + * device is about to be powered-down. + * + * @pwrctrl: PCI power control data. + */ +void pci_pwrctrl_device_unset_ready(struct pci_pwrctrl *pwrctrl) +{ + /* + * We don't have to delete the link here. Typically, this function + * is only called when the power control device is being detached. If + * it is being detached then the child PCI device must have already + * been unbound too or the device core wouldn't let us unbind. + */ + bus_unregister_notifier(&pci_bus_type, &pwrctrl->nb); +} +EXPORT_SYMBOL_GPL(pci_pwrctrl_device_unset_ready); + +static void devm_pci_pwrctrl_device_unset_ready(void *data) +{ + struct pci_pwrctrl *pwrctrl = data; + + pci_pwrctrl_device_unset_ready(pwrctrl); +} + +/** + * devm_pci_pwrctrl_device_set_ready - Managed variant of + * pci_pwrctrl_device_set_ready(). + * + * @dev: Device managing this pwrctrl provider. + * @pwrctrl: PCI power control data. + * + * Returns: + * 0 on success, negative error number on error. + */ +int devm_pci_pwrctrl_device_set_ready(struct device *dev, + struct pci_pwrctrl *pwrctrl) +{ + int ret; + + ret = pci_pwrctrl_device_set_ready(pwrctrl); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, + devm_pci_pwrctrl_device_unset_ready, + pwrctrl); +} +EXPORT_SYMBOL_GPL(devm_pci_pwrctrl_device_set_ready); + +MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@linaro.org>"); +MODULE_DESCRIPTION("PCI Device Power Control core driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/pwrctl/pci-pwrctl-pwrseq.c b/drivers/pci/pwrctrl/pci-pwrctrl-pwrseq.c index 0e6bd47671c2..4e664e7b8dd2 100644 --- a/drivers/pci/pwrctl/pci-pwrctl-pwrseq.c +++ b/drivers/pci/pwrctrl/pci-pwrctrl-pwrseq.c @@ -6,19 +6,19 @@ #include <linux/device.h> #include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/pci-pwrctl.h> +#include <linux/pci-pwrctrl.h> #include <linux/platform_device.h> #include <linux/property.h> #include <linux/pwrseq/consumer.h> #include <linux/slab.h> #include <linux/types.h> -struct pci_pwrctl_pwrseq_data { - struct pci_pwrctl ctx; +struct pci_pwrctrl_pwrseq_data { + struct pci_pwrctrl ctx; struct pwrseq_desc *pwrseq; }; -struct pci_pwrctl_pwrseq_pdata { +struct pci_pwrctrl_pwrseq_pdata { const char *target; /* * Called before doing anything else to perform device-specific @@ -27,7 +27,7 @@ struct pci_pwrctl_pwrseq_pdata { int (*validate_device)(struct device *dev); }; -static int pci_pwrctl_pwrseq_qcm_wcn_validate_device(struct device *dev) +static int pci_pwrctrl_pwrseq_qcm_wcn_validate_device(struct device *dev) { /* * Old device trees for some platforms already define wifi nodes for @@ -47,22 +47,22 @@ static int pci_pwrctl_pwrseq_qcm_wcn_validate_device(struct device *dev) return 0; } -static const struct pci_pwrctl_pwrseq_pdata pci_pwrctl_pwrseq_qcom_wcn_pdata = { +static const struct pci_pwrctrl_pwrseq_pdata pci_pwrctrl_pwrseq_qcom_wcn_pdata = { .target = "wlan", - .validate_device = pci_pwrctl_pwrseq_qcm_wcn_validate_device, + .validate_device = pci_pwrctrl_pwrseq_qcm_wcn_validate_device, }; -static void devm_pci_pwrctl_pwrseq_power_off(void *data) +static void devm_pci_pwrctrl_pwrseq_power_off(void *data) { struct pwrseq_desc *pwrseq = data; pwrseq_power_off(pwrseq); } -static int pci_pwrctl_pwrseq_probe(struct platform_device *pdev) +static int pci_pwrctrl_pwrseq_probe(struct platform_device *pdev) { - const struct pci_pwrctl_pwrseq_pdata *pdata; - struct pci_pwrctl_pwrseq_data *data; + const struct pci_pwrctrl_pwrseq_pdata *pdata; + struct pci_pwrctrl_pwrseq_data *data; struct device *dev = &pdev->dev; int ret; @@ -90,49 +90,49 @@ static int pci_pwrctl_pwrseq_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Failed to power-on the device\n"); - ret = devm_add_action_or_reset(dev, devm_pci_pwrctl_pwrseq_power_off, + ret = devm_add_action_or_reset(dev, devm_pci_pwrctrl_pwrseq_power_off, data->pwrseq); if (ret) return ret; - pci_pwrctl_init(&data->ctx, dev); + pci_pwrctrl_init(&data->ctx, dev); - ret = devm_pci_pwrctl_device_set_ready(dev, &data->ctx); + ret = devm_pci_pwrctrl_device_set_ready(dev, &data->ctx); if (ret) return dev_err_probe(dev, ret, - "Failed to register the pwrctl wrapper\n"); + "Failed to register the pwrctrl wrapper\n"); return 0; } -static const struct of_device_id pci_pwrctl_pwrseq_of_match[] = { +static const struct of_device_id pci_pwrctrl_pwrseq_of_match[] = { { /* ATH11K in QCA6390 package. */ .compatible = "pci17cb,1101", - .data = &pci_pwrctl_pwrseq_qcom_wcn_pdata, + .data = &pci_pwrctrl_pwrseq_qcom_wcn_pdata, }, { /* ATH11K in WCN6855 package. */ .compatible = "pci17cb,1103", - .data = &pci_pwrctl_pwrseq_qcom_wcn_pdata, + .data = &pci_pwrctrl_pwrseq_qcom_wcn_pdata, }, { /* ATH12K in WCN7850 package. */ .compatible = "pci17cb,1107", - .data = &pci_pwrctl_pwrseq_qcom_wcn_pdata, + .data = &pci_pwrctrl_pwrseq_qcom_wcn_pdata, }, { } }; -MODULE_DEVICE_TABLE(of, pci_pwrctl_pwrseq_of_match); +MODULE_DEVICE_TABLE(of, pci_pwrctrl_pwrseq_of_match); -static struct platform_driver pci_pwrctl_pwrseq_driver = { +static struct platform_driver pci_pwrctrl_pwrseq_driver = { .driver = { - .name = "pci-pwrctl-pwrseq", - .of_match_table = pci_pwrctl_pwrseq_of_match, + .name = "pci-pwrctrl-pwrseq", + .of_match_table = pci_pwrctrl_pwrseq_of_match, }, - .probe = pci_pwrctl_pwrseq_probe, + .probe = pci_pwrctrl_pwrseq_probe, }; -module_platform_driver(pci_pwrctl_pwrseq_driver); +module_platform_driver(pci_pwrctrl_pwrseq_driver); MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@linaro.org>"); MODULE_DESCRIPTION("Generic PCI Power Control module for power sequenced devices"); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 41dec625ed7b..76f4df75b08a 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -12,6 +12,7 @@ * file, where their drivers can use them. */ +#include <linux/align.h> #include <linux/bitfield.h> #include <linux/types.h> #include <linux/kernel.h> @@ -29,10 +30,23 @@ #include <linux/nvme.h> #include <linux/platform_data/x86/apple.h> #include <linux/pm_runtime.h> +#include <linux/sizes.h> #include <linux/suspend.h> #include <linux/switchtec.h> #include "pci.h" +static bool pcie_lbms_seen(struct pci_dev *dev, u16 lnksta) +{ + unsigned long count; + int ret; + + ret = pcie_lbms_count(dev, &count); + if (ret < 0) + return lnksta & PCI_EXP_LNKSTA_LBMS; + + return count > 0; +} + /* * Retrain the link of a downstream PCIe port by hand if necessary. * @@ -96,22 +110,16 @@ int pcie_failed_link_retrain(struct pci_dev *dev) pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &lnkctl2); pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); - if ((lnksta & (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_DLLLA)) == - PCI_EXP_LNKSTA_LBMS) { + if (!(lnksta & PCI_EXP_LNKSTA_DLLLA) && pcie_lbms_seen(dev, lnksta)) { u16 oldlnkctl2 = lnkctl2; pci_info(dev, "broken device, retraining non-functional downstream link at 2.5GT/s\n"); - lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; - lnkctl2 |= PCI_EXP_LNKCTL2_TLS_2_5GT; - pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, lnkctl2); - - ret = pcie_retrain_link(dev, false); + ret = pcie_set_target_speed(dev, PCIE_SPEED_2_5GT, false); if (ret) { pci_info(dev, "retraining failed\n"); - pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, - oldlnkctl2); - pcie_retrain_link(dev, true); + pcie_set_target_speed(dev, PCIE_LNKCTL2_TLS2SPEED(oldlnkctl2), + true); return ret; } @@ -125,11 +133,7 @@ int pcie_failed_link_retrain(struct pci_dev *dev) pci_info(dev, "removing 2.5GT/s downstream link speed restriction\n"); pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); - lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; - lnkctl2 |= lnkcap & PCI_EXP_LNKCAP_SLS; - pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, lnkctl2); - - ret = pcie_retrain_link(dev, false); + ret = pcie_set_target_speed(dev, PCIE_LNKCAP_SLS2SPEED(lnkcap), false); if (ret) { pci_info(dev, "retraining failed\n"); return ret; @@ -586,8 +590,7 @@ static void quirk_extend_bar_to_page(struct pci_dev *dev) const char *r_name = pci_resource_name(dev, i); if (r->flags & IORESOURCE_MEM && resource_size(r) < PAGE_SIZE) { - r->end = PAGE_SIZE - 1; - r->start = 0; + resource_set_range(r, 0, PAGE_SIZE); r->flags |= IORESOURCE_UNSET; pci_info(dev, "%s %pR: expanded to page size\n", r_name, r); @@ -604,10 +607,9 @@ static void quirk_s3_64M(struct pci_dev *dev) { struct resource *r = &dev->resource[0]; - if ((r->start & 0x3ffffff) || r->end != r->start + 0x3ffffff) { + if (!IS_ALIGNED(r->start, SZ_64M) || resource_size(r) != SZ_64M) { r->flags |= IORESOURCE_UNSET; - r->start = 0; - r->end = 0x3ffffff; + resource_set_range(r, 0, SZ_64M); } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_S3, PCI_DEVICE_ID_S3_868, quirk_s3_64M); @@ -1342,8 +1344,7 @@ static void quirk_dunord(struct pci_dev *dev) struct resource *r = &dev->resource[1]; r->flags |= IORESOURCE_UNSET; - r->start = 0; - r->end = 0xffffff; + resource_set_range(r, 0, SZ_16M); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DUNORD, PCI_DEVICE_ID_DUNORD_I3000, quirk_dunord); @@ -2340,8 +2341,7 @@ static void quirk_tc86c001_ide(struct pci_dev *dev) if (r->start & 0x8) { r->flags |= IORESOURCE_UNSET; - r->start = 0; - r->end = 0xf; + resource_set_range(r, 0, SZ_16); } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA_2, @@ -2369,8 +2369,7 @@ static void quirk_plx_pci9050(struct pci_dev *dev) pci_info(dev, "Re-allocating PLX PCI 9050 BAR %u to length 256 to avoid bit 7 bug\n", bar); r->flags |= IORESOURCE_UNSET; - r->start = 0; - r->end = 0xff; + resource_set_range(r, 0, SZ_256); } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050, @@ -3522,13 +3521,13 @@ static void quirk_intel_ntb(struct pci_dev *dev) if (rc) return; - dev->resource[2].end = dev->resource[2].start + ((u64) 1 << val) - 1; + resource_set_size(&dev->resource[2], (resource_size_t)1 << val); rc = pci_read_config_byte(dev, 0x00D1, &val); if (rc) return; - dev->resource[4].end = dev->resource[4].start + ((u64) 1 << val) - 1; + resource_set_size(&dev->resource[4], (resource_size_t)1 << val); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0e08, quirk_intel_ntb); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0e0d, quirk_intel_ntb); @@ -4996,18 +4995,21 @@ static int pci_quirk_brcm_acs(struct pci_dev *dev, u16 acs_flags) } /* - * Wangxun 10G/1G NICs have no ACS capability, and on multi-function - * devices, peer-to-peer transactions are not be used between the functions. - * So add an ACS quirk for below devices to isolate functions. + * Wangxun 40G/25G/10G/1G NICs have no ACS capability, but on + * multi-function devices, the hardware isolates the functions by + * directing all peer-to-peer traffic upstream as though PCI_ACS_RR and + * PCI_ACS_CR were set. * SFxxx 1G NICs(em). * RP1000/RP2000 10G NICs(sp). + * FF5xxx 40G/25G/10G NICs(aml). */ static int pci_quirk_wangxun_nic_acs(struct pci_dev *dev, u16 acs_flags) { switch (dev->device) { - case 0x0100 ... 0x010F: - case 0x1001: - case 0x2001: + case 0x0100 ... 0x010F: /* EM */ + case 0x1001: case 0x2001: /* SP */ + case 0x5010: case 0x5025: case 0x5040: /* AML */ + case 0x5110: case 0x5125: case 0x5140: /* AML */ return pci_acs_ctrl_enabled(acs_flags, PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF); } diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index e4ce1145aa3e..963b8d2855c1 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -17,37 +17,35 @@ static void pci_free_resources(struct pci_dev *dev) } } -static int pci_pwrctl_unregister(struct device *dev, void *data) +static void pci_pwrctrl_unregister(struct device *dev) { - struct device_node *pci_node = data, *plat_node = dev_of_node(dev); + struct platform_device *pdev; - if (dev_is_platform(dev) && plat_node && plat_node == pci_node) { - of_device_unregister(to_platform_device(dev)); - of_node_clear_flag(plat_node, OF_POPULATED); - } + pdev = of_find_device_by_node(dev_of_node(dev)); + if (!pdev) + return; - return 0; + of_device_unregister(pdev); + of_node_clear_flag(dev_of_node(dev), OF_POPULATED); } static void pci_stop_dev(struct pci_dev *dev) { pci_pme_active(dev, false); - if (pci_dev_is_added(dev)) { - device_for_each_child(dev->dev.parent, dev_of_node(&dev->dev), - pci_pwrctl_unregister); - device_release_driver(&dev->dev); - pci_proc_detach_device(dev); - pci_remove_sysfs_dev_files(dev); - of_pci_remove_node(dev); + if (!pci_dev_test_and_clear_added(dev)) + return; - pci_dev_assign_added(dev, false); - } + pci_pwrctrl_unregister(&dev->dev); + device_release_driver(&dev->dev); + pci_proc_detach_device(dev); + pci_remove_sysfs_dev_files(dev); + of_pci_remove_node(dev); } static void pci_destroy_dev(struct pci_dev *dev) { - if (!dev->dev.kobj.parent) + if (pci_dev_test_and_set_removed(dev)) return; pci_npem_remove(dev); diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 23082bc0ca37..5e00cecf1f1a 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -134,6 +134,7 @@ static void pdev_sort_resources(struct pci_dev *dev, struct list_head *head) int i; pci_dev_for_each_resource(dev, r, i) { + const char *r_name = pci_resource_name(dev, i); struct pci_dev_resource *dev_res, *tmp; resource_size_t r_align; struct list_head *n; @@ -146,8 +147,8 @@ static void pdev_sort_resources(struct pci_dev *dev, struct list_head *head) r_align = pci_resource_alignment(dev, r); if (!r_align) { - pci_warn(dev, "BAR %d: %pR has bogus alignment\n", - i, r); + pci_warn(dev, "%s %pR: alignment must not be zero\n", + r_name, r); continue; } @@ -246,8 +247,7 @@ static void reassign_resources_sorted(struct list_head *realloc_head, add_size = add_res->add_size; align = add_res->min_align; if (!resource_size(res)) { - res->start = align; - res->end = res->start + add_size - 1; + resource_set_range(res, align, add_size); if (pci_assign_resource(add_res->dev, idx)) reset_resource(res); } else { @@ -938,8 +938,7 @@ static void pbus_size_io(struct pci_bus *bus, resource_size_t min_size, return; } - b_res->start = min_align; - b_res->end = b_res->start + size0 - 1; + resource_set_range(b_res, min_align, size0); b_res->flags |= IORESOURCE_STARTALIGN; if (bus->self && size1 > size0 && realloc_head) { add_to_list(realloc_head, bus->self, b_res, size1-size0, @@ -1202,8 +1201,7 @@ static void pci_bus_size_cardbus(struct pci_bus *bus, * Reserve some resources for CardBus. We reserve a fixed amount * of bus space for CardBus bridges. */ - b_res->start = pci_cardbus_io_size; - b_res->end = b_res->start + pci_cardbus_io_size - 1; + resource_set_range(b_res, pci_cardbus_io_size, pci_cardbus_io_size); b_res->flags |= IORESOURCE_IO | IORESOURCE_STARTALIGN; if (realloc_head) { b_res->end -= pci_cardbus_io_size; @@ -1215,8 +1213,7 @@ handle_b_res_1: b_res = &bridge->resource[PCI_CB_BRIDGE_IO_1_WINDOW]; if (b_res->parent) goto handle_b_res_2; - b_res->start = pci_cardbus_io_size; - b_res->end = b_res->start + pci_cardbus_io_size - 1; + resource_set_range(b_res, pci_cardbus_io_size, pci_cardbus_io_size); b_res->flags |= IORESOURCE_IO | IORESOURCE_STARTALIGN; if (realloc_head) { b_res->end -= pci_cardbus_io_size; @@ -1249,8 +1246,8 @@ handle_b_res_2: * Otherwise, allocate one region of twice the size. */ if (ctrl & PCI_CB_BRIDGE_CTL_PREFETCH_MEM0) { - b_res->start = pci_cardbus_mem_size; - b_res->end = b_res->start + pci_cardbus_mem_size - 1; + resource_set_range(b_res, pci_cardbus_mem_size, + pci_cardbus_mem_size); b_res->flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH | IORESOURCE_STARTALIGN; if (realloc_head) { @@ -1267,8 +1264,7 @@ handle_b_res_3: b_res = &bridge->resource[PCI_CB_BRIDGE_MEM_1_WINDOW]; if (b_res->parent) goto handle_done; - b_res->start = pci_cardbus_mem_size; - b_res->end = b_res->start + b_res_3_size - 1; + resource_set_range(b_res, pci_cardbus_mem_size, b_res_3_size); b_res->flags |= IORESOURCE_MEM | IORESOURCE_STARTALIGN; if (realloc_head) { b_res->end -= b_res_3_size; @@ -1847,7 +1843,7 @@ static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res, return; } - res->end = res->start + new_size - 1; + resource_set_size(res, new_size); /* If the resource is part of the add_list, remove it now */ if (add_list) @@ -1899,6 +1895,9 @@ static void remove_dev_resources(struct pci_dev *dev, struct resource *io, } } +#define ALIGN_DOWN_IF_NONZERO(addr, align) \ + ((align) ? ALIGN_DOWN((addr), (align)) : (addr)) + /* * io, mmio and mmio_pref contain the total amount of bridge window space * available. This includes the minimal space needed to cover all the @@ -2010,8 +2009,7 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, * what is available). */ align = pci_resource_alignment(dev, res); - io.end = align ? io.start + ALIGN_DOWN(io_per_b, align) - 1 - : io.start + io_per_b - 1; + resource_set_size(&io, ALIGN_DOWN_IF_NONZERO(io_per_b, align)); /* * The x_per_b holds the extra resource space that can be @@ -2023,15 +2021,14 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, res = &dev->resource[PCI_BRIDGE_MEM_WINDOW]; align = pci_resource_alignment(dev, res); - mmio.end = align ? mmio.start + ALIGN_DOWN(mmio_per_b, align) - 1 - : mmio.start + mmio_per_b - 1; + resource_set_size(&mmio, + ALIGN_DOWN_IF_NONZERO(mmio_per_b,align)); mmio.start -= resource_size(res); res = &dev->resource[PCI_BRIDGE_PREF_MEM_WINDOW]; align = pci_resource_alignment(dev, res); - mmio_pref.end = align ? mmio_pref.start + - ALIGN_DOWN(mmio_pref_per_b, align) - 1 - : mmio_pref.start + mmio_pref_per_b - 1; + resource_set_size(&mmio_pref, + ALIGN_DOWN_IF_NONZERO(mmio_pref_per_b, align)); mmio_pref.start -= resource_size(res); pci_bus_distribute_available_resources(b, add_list, io, mmio, diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddfd46..ca14576bf2bf 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -211,8 +211,7 @@ static int pci_revert_fw_address(struct resource *res, struct pci_dev *dev, start = res->start; end = res->end; - res->start = fw_addr; - res->end = res->start + size - 1; + resource_set_range(res, fw_addr, size); res->flags &= ~IORESOURCE_UNSET; root = pci_find_parent_resource(dev, res); @@ -463,7 +462,7 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) if (ret) return ret; - res->end = res->start + pci_rebar_size_to_bytes(size) - 1; + resource_set_size(res, pci_rebar_size_to_bytes(size)); /* Check if the new config works by trying to assign everything. */ if (dev->bus->self) { @@ -475,7 +474,7 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size) error_resize: pci_rebar_set_size(dev, resno, old); - res->end = res->start + pci_rebar_size_to_bytes(old) - 1; + resource_set_size(res, pci_rebar_size_to_bytes(old)); return ret; } EXPORT_SYMBOL(pci_resize_resource); diff --git a/drivers/pci/slot.c b/drivers/pci/slot.c index 0f87cade10f7..36b44be0489d 100644 --- a/drivers/pci/slot.c +++ b/drivers/pci/slot.c @@ -79,6 +79,7 @@ static void pci_slot_release(struct kobject *kobj) up_read(&pci_bus_sem); list_del(&slot->list); + pci_bus_put(slot->bus); kfree(slot); } @@ -244,12 +245,13 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, slot = get_slot(parent, slot_nr); if (slot) { if (hotplug) { - if ((err = slot->hotplug ? -EBUSY : 0) - || (err = rename_slot(slot, name))) { - kobject_put(&slot->kobj); - slot = NULL; - goto err; + if (slot->hotplug) { + err = -EBUSY; + goto put_slot; } + err = rename_slot(slot, name); + if (err) + goto put_slot; } goto out; } @@ -261,7 +263,7 @@ placeholder: goto err; } - slot->bus = parent; + slot->bus = pci_bus_get(parent); slot->number = slot_nr; slot->kobj.kset = pci_slots_kset; @@ -269,6 +271,7 @@ placeholder: slot_name = make_slot_name(name); if (!slot_name) { err = -ENOMEM; + pci_bus_put(slot->bus); kfree(slot); goto err; } @@ -278,10 +281,8 @@ placeholder: err = kobject_init_and_add(&slot->kobj, &pci_slot_ktype, NULL, "%s", slot_name); - if (err) { - kobject_put(&slot->kobj); - goto err; - } + if (err) + goto put_slot; down_read(&pci_bus_sem); list_for_each_entry(dev, &parent->devices, bus_list) @@ -296,6 +297,9 @@ out: kfree(slot_name); mutex_unlock(&pci_slot_mutex); return slot; + +put_slot: + kobject_put(&slot->kobj); err: slot = ERR_PTR(err); goto out; diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c new file mode 100644 index 000000000000..1e604fbbda65 --- /dev/null +++ b/drivers/pci/tph.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TPH (TLP Processing Hints) support + * + * Copyright (C) 2024 Advanced Micro Devices, Inc. + * Eric Van Tassell <Eric.VanTassell@amd.com> + * Wei Huang <wei.huang2@amd.com> + */ +#include <linux/pci.h> +#include <linux/pci-acpi.h> +#include <linux/msi.h> +#include <linux/bitfield.h> +#include <linux/pci-tph.h> + +#include "pci.h" + +/* System-wide TPH disabled */ +static bool pci_tph_disabled; + +#ifdef CONFIG_ACPI +/* + * The st_info struct defines the Steering Tag (ST) info returned by the + * firmware PCI ACPI _DSM method (rev=0x7, func=0xF, "_DSM to Query Cache + * Locality TPH Features"), as specified in the approved ECN for PCI Firmware + * Spec and available at https://members.pcisig.com/wg/PCI-SIG/document/15470. + * + * @vm_st_valid: 8-bit ST for volatile memory is valid + * @vm_xst_valid: 16-bit extended ST for volatile memory is valid + * @vm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied + * @vm_st: 8-bit ST for volatile mem + * @vm_xst: 16-bit extended ST for volatile mem + * @pm_st_valid: 8-bit ST for persistent memory is valid + * @pm_xst_valid: 16-bit extended ST for persistent memory is valid + * @pm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied + * @pm_st: 8-bit ST for persistent mem + * @pm_xst: 16-bit extended ST for persistent mem + */ +union st_info { + struct { + u64 vm_st_valid : 1; + u64 vm_xst_valid : 1; + u64 vm_ph_ignore : 1; + u64 rsvd1 : 5; + u64 vm_st : 8; + u64 vm_xst : 16; + u64 pm_st_valid : 1; + u64 pm_xst_valid : 1; + u64 pm_ph_ignore : 1; + u64 rsvd2 : 5; + u64 pm_st : 8; + u64 pm_xst : 16; + }; + u64 value; +}; + +static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type, + union st_info *info) +{ + switch (req_type) { + case PCI_TPH_REQ_TPH_ONLY: /* 8-bit tag */ + switch (mem_type) { + case TPH_MEM_TYPE_VM: + if (info->vm_st_valid) + return info->vm_st; + break; + case TPH_MEM_TYPE_PM: + if (info->pm_st_valid) + return info->pm_st; + break; + } + break; + case PCI_TPH_REQ_EXT_TPH: /* 16-bit tag */ + switch (mem_type) { + case TPH_MEM_TYPE_VM: + if (info->vm_xst_valid) + return info->vm_xst; + break; + case TPH_MEM_TYPE_PM: + if (info->pm_xst_valid) + return info->pm_xst; + break; + } + break; + default: + return 0; + } + + return 0; +} + +#define TPH_ST_DSM_FUNC_INDEX 0xF +static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid, + union st_info *st_out) +{ + union acpi_object arg3[3], in_obj, *out_obj; + + if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 7, + BIT(TPH_ST_DSM_FUNC_INDEX))) + return AE_ERROR; + + /* DWORD: feature ID (0 for processor cache ST query) */ + arg3[0].integer.type = ACPI_TYPE_INTEGER; + arg3[0].integer.value = 0; + + /* DWORD: target UID */ + arg3[1].integer.type = ACPI_TYPE_INTEGER; + arg3[1].integer.value = cpu_uid; + + /* QWORD: properties, all 0's */ + arg3[2].integer.type = ACPI_TYPE_INTEGER; + arg3[2].integer.value = 0; + + in_obj.type = ACPI_TYPE_PACKAGE; + in_obj.package.count = ARRAY_SIZE(arg3); + in_obj.package.elements = arg3; + + out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 7, + TPH_ST_DSM_FUNC_INDEX, &in_obj); + if (!out_obj) + return AE_ERROR; + + if (out_obj->type != ACPI_TYPE_BUFFER) { + ACPI_FREE(out_obj); + return AE_ERROR; + } + + st_out->value = *((u64 *)(out_obj->buffer.pointer)); + + ACPI_FREE(out_obj); + + return AE_OK; +} +#endif + +/* Update the TPH Requester Enable field of TPH Control Register */ +static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type) +{ + u32 reg; + + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, ®); + + reg &= ~PCI_TPH_CTRL_REQ_EN_MASK; + reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, req_type); + + pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg); +} + +static u8 get_st_modes(struct pci_dev *pdev) +{ + u32 reg; + + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); + reg &= PCI_TPH_CAP_ST_NS | PCI_TPH_CAP_ST_IV | PCI_TPH_CAP_ST_DS; + + return reg; +} + +static u32 get_st_table_loc(struct pci_dev *pdev) +{ + u32 reg; + + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); + + return FIELD_GET(PCI_TPH_CAP_LOC_MASK, reg); +} + +/* + * Return the size of ST table. If ST table is not in TPH Requester Extended + * Capability space, return 0. Otherwise return the ST Table Size + 1. + */ +static u16 get_st_table_size(struct pci_dev *pdev) +{ + u32 reg; + u32 loc; + + /* Check ST table location first */ + loc = get_st_table_loc(pdev); + + /* Convert loc to match with PCI_TPH_LOC_* defined in pci_regs.h */ + loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc); + if (loc != PCI_TPH_LOC_CAP) + return 0; + + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); + + return FIELD_GET(PCI_TPH_CAP_ST_MASK, reg) + 1; +} + +/* Return device's Root Port completer capability */ +static u8 get_rp_completer_type(struct pci_dev *pdev) +{ + struct pci_dev *rp; + u32 reg; + int ret; + + rp = pcie_find_root_port(pdev); + if (!rp) + return 0; + + ret = pcie_capability_read_dword(rp, PCI_EXP_DEVCAP2, ®); + if (ret) + return 0; + + return FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg); +} + +/* Write ST to MSI-X vector control reg - Return 0 if OK, otherwise -errno */ +static int write_tag_to_msix(struct pci_dev *pdev, int msix_idx, u16 tag) +{ +#ifdef CONFIG_PCI_MSI + struct msi_desc *msi_desc = NULL; + void __iomem *vec_ctrl; + u32 val; + int err = 0; + + msi_lock_descs(&pdev->dev); + + /* Find the msi_desc entry with matching msix_idx */ + msi_for_each_desc(msi_desc, &pdev->dev, MSI_DESC_ASSOCIATED) { + if (msi_desc->msi_index == msix_idx) + break; + } + + if (!msi_desc) { + err = -ENXIO; + goto err_out; + } + + /* Get the vector control register (offset 0xc) pointed by msix_idx */ + vec_ctrl = pdev->msix_base + msix_idx * PCI_MSIX_ENTRY_SIZE; + vec_ctrl += PCI_MSIX_ENTRY_VECTOR_CTRL; + + val = readl(vec_ctrl); + val &= ~PCI_MSIX_ENTRY_CTRL_ST; + val |= FIELD_PREP(PCI_MSIX_ENTRY_CTRL_ST, tag); + writel(val, vec_ctrl); + + /* Read back to flush the update */ + val = readl(vec_ctrl); + +err_out: + msi_unlock_descs(&pdev->dev); + return err; +#else + return -ENODEV; +#endif +} + +/* Write tag to ST table - Return 0 if OK, otherwise -errno */ +static int write_tag_to_st_table(struct pci_dev *pdev, int index, u16 tag) +{ + int st_table_size; + int offset; + + /* Check if index is out of bound */ + st_table_size = get_st_table_size(pdev); + if (index >= st_table_size) + return -ENXIO; + + offset = pdev->tph_cap + PCI_TPH_BASE_SIZEOF + index * sizeof(u16); + + return pci_write_config_word(pdev, offset, tag); +} + +/** + * pcie_tph_get_cpu_st() - Retrieve Steering Tag for a target memory associated + * with a specific CPU + * @pdev: PCI device + * @mem_type: target memory type (volatile or persistent RAM) + * @cpu_uid: associated CPU id + * @tag: Steering Tag to be returned + * + * Return the Steering Tag for a target memory that is associated with a + * specific CPU as indicated by cpu_uid. + * + * Return: 0 if success, otherwise negative value (-errno) + */ +int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type mem_type, + unsigned int cpu_uid, u16 *tag) +{ +#ifdef CONFIG_ACPI + struct pci_dev *rp; + acpi_handle rp_acpi_handle; + union st_info info; + + rp = pcie_find_root_port(pdev); + if (!rp || !rp->bus || !rp->bus->bridge) + return -ENODEV; + + rp_acpi_handle = ACPI_HANDLE(rp->bus->bridge); + + if (tph_invoke_dsm(rp_acpi_handle, cpu_uid, &info) != AE_OK) { + *tag = 0; + return -EINVAL; + } + + *tag = tph_extract_tag(mem_type, pdev->tph_req_type, &info); + + pci_dbg(pdev, "get steering tag: mem_type=%s, cpu_uid=%d, tag=%#04x\n", + (mem_type == TPH_MEM_TYPE_VM) ? "volatile" : "persistent", + cpu_uid, *tag); + + return 0; +#else + return -ENODEV; +#endif +} +EXPORT_SYMBOL(pcie_tph_get_cpu_st); + +/** + * pcie_tph_set_st_entry() - Set Steering Tag in the ST table entry + * @pdev: PCI device + * @index: ST table entry index + * @tag: Steering Tag to be written + * + * Figure out the proper location of ST table, either in the MSI-X table or + * in the TPH Extended Capability space, and write the Steering Tag into + * the ST entry pointed by index. + * + * Return: 0 if success, otherwise negative value (-errno) + */ +int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag) +{ + u32 loc; + int err = 0; + + if (!pdev->tph_cap) + return -EINVAL; + + if (!pdev->tph_enabled) + return -EINVAL; + + /* No need to write tag if device is in "No ST Mode" */ + if (pdev->tph_mode == PCI_TPH_ST_NS_MODE) + return 0; + + /* + * Disable TPH before updating ST to avoid potential instability as + * cautioned in PCIe r6.2, sec 6.17.3, "ST Modes of Operation" + */ + set_ctrl_reg_req_en(pdev, PCI_TPH_REQ_DISABLE); + + loc = get_st_table_loc(pdev); + /* Convert loc to match with PCI_TPH_LOC_* */ + loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc); + + switch (loc) { + case PCI_TPH_LOC_MSIX: + err = write_tag_to_msix(pdev, index, tag); + break; + case PCI_TPH_LOC_CAP: + err = write_tag_to_st_table(pdev, index, tag); + break; + default: + err = -EINVAL; + } + + if (err) { + pcie_disable_tph(pdev); + return err; + } + + set_ctrl_reg_req_en(pdev, pdev->tph_mode); + + pci_dbg(pdev, "set steering tag: %s table, index=%d, tag=%#04x\n", + (loc == PCI_TPH_LOC_MSIX) ? "MSI-X" : "ST", index, tag); + + return 0; +} +EXPORT_SYMBOL(pcie_tph_set_st_entry); + +/** + * pcie_disable_tph - Turn off TPH support for device + * @pdev: PCI device + * + * Return: none + */ +void pcie_disable_tph(struct pci_dev *pdev) +{ + if (!pdev->tph_cap) + return; + + if (!pdev->tph_enabled) + return; + + pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, 0); + + pdev->tph_mode = 0; + pdev->tph_req_type = 0; + pdev->tph_enabled = 0; +} +EXPORT_SYMBOL(pcie_disable_tph); + +/** + * pcie_enable_tph - Enable TPH support for device using a specific ST mode + * @pdev: PCI device + * @mode: ST mode to enable. Current supported modes include: + * + * - PCI_TPH_ST_NS_MODE: NO ST Mode + * - PCI_TPH_ST_IV_MODE: Interrupt Vector Mode + * - PCI_TPH_ST_DS_MODE: Device Specific Mode + * + * Check whether the mode is actually supported by the device before enabling + * and return an error if not. Additionally determine what types of requests, + * TPH or extended TPH, can be issued by the device based on its TPH requester + * capability and the Root Port's completer capability. + * + * Return: 0 on success, otherwise negative value (-errno) + */ +int pcie_enable_tph(struct pci_dev *pdev, int mode) +{ + u32 reg; + u8 dev_modes; + u8 rp_req_type; + + /* Honor "notph" kernel parameter */ + if (pci_tph_disabled) + return -EINVAL; + + if (!pdev->tph_cap) + return -EINVAL; + + if (pdev->tph_enabled) + return -EBUSY; + + /* Sanitize and check ST mode compatibility */ + mode &= PCI_TPH_CTRL_MODE_SEL_MASK; + dev_modes = get_st_modes(pdev); + if (!((1 << mode) & dev_modes)) + return -EINVAL; + + pdev->tph_mode = mode; + + /* Get req_type supported by device and its Root Port */ + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); + if (FIELD_GET(PCI_TPH_CAP_EXT_TPH, reg)) + pdev->tph_req_type = PCI_TPH_REQ_EXT_TPH; + else + pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY; + + rp_req_type = get_rp_completer_type(pdev); + + /* Final req_type is the smallest value of two */ + pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type); + + if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE) + return -EINVAL; + + /* Write them into TPH control register */ + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, ®); + + reg &= ~PCI_TPH_CTRL_MODE_SEL_MASK; + reg |= FIELD_PREP(PCI_TPH_CTRL_MODE_SEL_MASK, pdev->tph_mode); + + reg &= ~PCI_TPH_CTRL_REQ_EN_MASK; + reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, pdev->tph_req_type); + + pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg); + + pdev->tph_enabled = 1; + + return 0; +} +EXPORT_SYMBOL(pcie_enable_tph); + +void pci_restore_tph_state(struct pci_dev *pdev) +{ + struct pci_cap_saved_state *save_state; + int num_entries, i, offset; + u16 *st_entry; + u32 *cap; + + if (!pdev->tph_cap) + return; + + if (!pdev->tph_enabled) + return; + + save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH); + if (!save_state) + return; + + /* Restore control register and all ST entries */ + cap = &save_state->cap.data[0]; + pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, *cap++); + st_entry = (u16 *)cap; + offset = PCI_TPH_BASE_SIZEOF; + num_entries = get_st_table_size(pdev); + for (i = 0; i < num_entries; i++) { + pci_write_config_word(pdev, pdev->tph_cap + offset, + *st_entry++); + offset += sizeof(u16); + } +} + +void pci_save_tph_state(struct pci_dev *pdev) +{ + struct pci_cap_saved_state *save_state; + int num_entries, i, offset; + u16 *st_entry; + u32 *cap; + + if (!pdev->tph_cap) + return; + + if (!pdev->tph_enabled) + return; + + save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH); + if (!save_state) + return; + + /* Save control register */ + cap = &save_state->cap.data[0]; + pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, cap++); + + /* Save all ST entries in extended capability structure */ + st_entry = (u16 *)cap; + offset = PCI_TPH_BASE_SIZEOF; + num_entries = get_st_table_size(pdev); + for (i = 0; i < num_entries; i++) { + pci_read_config_word(pdev, pdev->tph_cap + offset, + st_entry++); + offset += sizeof(u16); + } +} + +void pci_no_tph(void) +{ + pci_tph_disabled = true; + + pr_info("PCIe TPH is disabled\n"); +} + +void pci_tph_init(struct pci_dev *pdev) +{ + int num_entries; + u32 save_size; + + pdev->tph_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_TPH); + if (!pdev->tph_cap) + return; + + num_entries = get_st_table_size(pdev); + save_size = sizeof(u32) + num_entries * sizeof(u16); + pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_TPH, save_size); +} diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index 955e4e38477e..83962a114dc9 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -329,7 +329,8 @@ config STM32_RPROC config TI_K3_DSP_REMOTEPROC tristate "TI K3 DSP remoteproc support" - depends on ARCH_K3 + depends on ARCH_K3 || COMPILE_TEST + depends on TI_SCI_PROTOCOL || (COMPILE_TEST && TI_SCI_PROTOCOL=n) depends on OMAP2PLUS_MBOX help Say m here to support TI's C66x and C71x DSP remote processor @@ -341,9 +342,9 @@ config TI_K3_DSP_REMOTEPROC config TI_K3_M4_REMOTEPROC tristate "TI K3 M4 remoteproc support" - depends on ARCH_OMAP2PLUS || ARCH_K3 - select MAILBOX - select OMAP2PLUS_MBOX + depends on ARCH_K3 || COMPILE_TEST + depends on TI_SCI_PROTOCOL || (COMPILE_TEST && TI_SCI_PROTOCOL=n) + depends on OMAP2PLUS_MBOX help Say m here to support TI's M4 remote processor subsystems on various TI K3 family of SoCs through the remote processor @@ -354,7 +355,8 @@ config TI_K3_M4_REMOTEPROC config TI_K3_R5_REMOTEPROC tristate "TI K3 R5 remoteproc support" - depends on ARCH_K3 + depends on ARCH_K3 || COMPILE_TEST + depends on TI_SCI_PROTOCOL || (COMPILE_TEST && TI_SCI_PROTOCOL=n) depends on OMAP2PLUS_MBOX help Say m here to support TI's R5F remote processor subsystems diff --git a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c index 8770d0cf1255..93031f0867d1 100644 --- a/drivers/remoteproc/da8xx_remoteproc.c +++ b/drivers/remoteproc/da8xx_remoteproc.c @@ -251,10 +251,8 @@ static int da8xx_rproc_probe(struct platform_device *pdev) return irq; irq_data = irq_get_irq_data(irq); - if (!irq_data) { - dev_err(dev, "irq_get_irq_data(%d): NULL\n", irq); - return -EINVAL; - } + if (!irq_data) + return dev_err_probe(dev, -EINVAL, "irq_get_irq_data(%d): NULL\n", irq); bootreg = devm_platform_ioremap_resource_byname(pdev, "host1cfg"); if (IS_ERR(bootreg)) @@ -265,28 +263,17 @@ static int da8xx_rproc_probe(struct platform_device *pdev) return PTR_ERR(chipsig); dsp_clk = devm_clk_get(dev, NULL); - if (IS_ERR(dsp_clk)) { - dev_err(dev, "clk_get error: %ld\n", PTR_ERR(dsp_clk)); - - return PTR_ERR(dsp_clk); - } + if (IS_ERR(dsp_clk)) + return dev_err_probe(dev, PTR_ERR(dsp_clk), "clk_get error\n"); dsp_reset = devm_reset_control_get_exclusive(dev, NULL); - if (IS_ERR(dsp_reset)) { - if (PTR_ERR(dsp_reset) != -EPROBE_DEFER) - dev_err(dev, "unable to get reset control: %ld\n", - PTR_ERR(dsp_reset)); - - return PTR_ERR(dsp_reset); - } + if (IS_ERR(dsp_reset)) + return dev_err_probe(dev, PTR_ERR(dsp_reset), "unable to get reset control\n"); if (dev->of_node) { ret = of_reserved_mem_device_init(dev); - if (ret) { - dev_err(dev, "device does not have specific CMA pool: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "device does not have specific CMA pool\n"); } rproc = rproc_alloc(dev, "dsp", &da8xx_rproc_ops, da8xx_fw_name, @@ -378,7 +365,7 @@ MODULE_DEVICE_TABLE(of, davinci_rproc_of_match); static struct platform_driver da8xx_rproc_driver = { .probe = da8xx_rproc_probe, - .remove_new = da8xx_rproc_remove, + .remove = da8xx_rproc_remove, .driver = { .name = "davinci-rproc", .of_match_table = of_match_ptr(davinci_rproc_of_match), diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 376187ad5754..ea5024919c2f 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -1258,7 +1258,7 @@ MODULE_DEVICE_TABLE(of, imx_dsp_rproc_of_match); static struct platform_driver imx_dsp_rproc_driver = { .probe = imx_dsp_rproc_probe, - .remove_new = imx_dsp_rproc_remove, + .remove = imx_dsp_rproc_remove, .driver = { .name = "imx-dsp-rproc", .of_match_table = imx_dsp_rproc_of_match, diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 800015ff7ff9..74299af1d7f1 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -1198,7 +1198,7 @@ MODULE_DEVICE_TABLE(of, imx_rproc_of_match); static struct platform_driver imx_rproc_driver = { .probe = imx_rproc_probe, - .remove_new = imx_rproc_remove, + .remove = imx_rproc_remove, .driver = { .name = "imx-rproc", .of_match_table = imx_rproc_of_match, diff --git a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c index 8f0f7a4cfef2..6e54093d1732 100644 --- a/drivers/remoteproc/keystone_remoteproc.c +++ b/drivers/remoteproc/keystone_remoteproc.c @@ -490,7 +490,7 @@ MODULE_DEVICE_TABLE(of, keystone_rproc_of_match); static struct platform_driver keystone_rproc_driver = { .probe = keystone_rproc_probe, - .remove_new = keystone_rproc_remove, + .remove = keystone_rproc_remove, .driver = { .name = "keystone-rproc", .of_match_table = keystone_rproc_of_match, diff --git a/drivers/remoteproc/meson_mx_ao_arc.c b/drivers/remoteproc/meson_mx_ao_arc.c index f6744b538323..7dfdf11b0036 100644 --- a/drivers/remoteproc/meson_mx_ao_arc.c +++ b/drivers/remoteproc/meson_mx_ao_arc.c @@ -246,7 +246,7 @@ MODULE_DEVICE_TABLE(of, meson_mx_ao_arc_rproc_match); static struct platform_driver meson_mx_ao_arc_rproc_driver = { .probe = meson_mx_ao_arc_rproc_probe, - .remove_new = meson_mx_ao_arc_rproc_remove, + .remove = meson_mx_ao_arc_rproc_remove, .driver = { .name = "meson-mx-ao-arc-rproc", .of_match_table = meson_mx_ao_arc_rproc_match, diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index e744c07507ee..0f4a7065d0bd 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -1521,7 +1521,7 @@ MODULE_DEVICE_TABLE(of, mtk_scp_of_match); static struct platform_driver mtk_scp_driver = { .probe = scp_probe, - .remove_new = scp_remove, + .remove = scp_remove, .driver = { .name = "mtk-scp", .of_match_table = mtk_scp_of_match, diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c index 327f0c7ee3d6..1656574b7317 100644 --- a/drivers/remoteproc/pru_rproc.c +++ b/drivers/remoteproc/pru_rproc.c @@ -1132,7 +1132,7 @@ static struct platform_driver pru_rproc_driver = { .suppress_bind_attrs = true, }, .probe = pru_rproc_probe, - .remove_new = pru_rproc_remove, + .remove = pru_rproc_remove, }; module_platform_driver(pru_rproc_driver); diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index 572dcb0f055b..94af77baa7a1 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -534,15 +534,11 @@ static const struct rproc_ops adsp_ops = { static int adsp_init_clock(struct qcom_adsp *adsp, const char **clk_ids) { int num_clks = 0; - int i, ret; + int i; adsp->xo = devm_clk_get(adsp->dev, "xo"); - if (IS_ERR(adsp->xo)) { - ret = PTR_ERR(adsp->xo); - if (ret != -EPROBE_DEFER) - dev_err(adsp->dev, "failed to get xo clock"); - return ret; - } + if (IS_ERR(adsp->xo)) + return dev_err_probe(adsp->dev, PTR_ERR(adsp->xo), "failed to get xo clock"); for (i = 0; clk_ids[i]; i++) num_clks++; @@ -708,10 +704,9 @@ static int adsp_probe(struct platform_device *pdev) return ret; ret = qcom_rproc_pds_attach(adsp, desc->pd_names, desc->num_pds); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to attach proxy power domains\n"); - return ret; - } + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Failed to attach proxy power domains\n"); ret = adsp_init_reset(adsp); if (ret) @@ -734,15 +729,22 @@ static int adsp_probe(struct platform_device *pdev) desc->ssctl_id); if (IS_ERR(adsp->sysmon)) { ret = PTR_ERR(adsp->sysmon); - goto disable_pm; + goto deinit_remove_glink_pdm_ssr; } ret = rproc_add(rproc); if (ret) - goto disable_pm; + goto remove_sysmon; return 0; +remove_sysmon: + qcom_remove_sysmon_subdev(adsp->sysmon); +deinit_remove_glink_pdm_ssr: + qcom_q6v5_deinit(&adsp->q6v5); + qcom_remove_glink_subdev(rproc, &adsp->glink_subdev); + qcom_remove_pdm_subdev(rproc, &adsp->pdm_subdev); + qcom_remove_ssr_subdev(rproc, &adsp->ssr_subdev); disable_pm: qcom_rproc_pds_detach(adsp); @@ -840,7 +842,7 @@ MODULE_DEVICE_TABLE(of, adsp_of_match); static struct platform_driver adsp_pil_driver = { .probe = adsp_probe, - .remove_new = adsp_remove, + .remove = adsp_remove, .driver = { .name = "qcom_q6v5_adsp", .of_match_table = adsp_of_match, diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 2a42215ce8e0..e78bd986dc3f 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -261,7 +261,6 @@ enum { static int q6v5_regulator_init(struct device *dev, struct reg_info *regs, const struct qcom_mss_reg_res *reg_res) { - int rc; int i; if (!reg_res) @@ -269,13 +268,10 @@ static int q6v5_regulator_init(struct device *dev, struct reg_info *regs, for (i = 0; reg_res[i].supply; i++) { regs[i].reg = devm_regulator_get(dev, reg_res[i].supply); - if (IS_ERR(regs[i].reg)) { - rc = PTR_ERR(regs[i].reg); - if (rc != -EPROBE_DEFER) - dev_err(dev, "Failed to get %s\n regulator", - reg_res[i].supply); - return rc; - } + if (IS_ERR(regs[i].reg)) + return dev_err_probe(dev, PTR_ERR(regs[i].reg), + "Failed to get %s\n regulator", + reg_res[i].supply); regs[i].uV = reg_res[i].uV; regs[i].uA = reg_res[i].uA; @@ -1162,6 +1158,9 @@ static int q6v5_mba_load(struct q6v5 *qproc) goto disable_active_clks; } + if (qproc->has_mba_logs) + qcom_pil_info_store("mba", qproc->mba_phys, MBA_LOG_SIZE); + writel(qproc->mba_phys, qproc->rmb_base + RMB_MBA_IMAGE_REG); if (qproc->dp_size) { writel(qproc->mba_phys + SZ_1M, qproc->rmb_base + RMB_PMI_CODE_START_REG); @@ -1172,9 +1171,6 @@ static int q6v5_mba_load(struct q6v5 *qproc) if (ret) goto reclaim_mba; - if (qproc->has_mba_logs) - qcom_pil_info_store("mba", qproc->mba_phys, MBA_LOG_SIZE); - ret = q6v5_rmb_mba_wait(qproc, 0, 5000); if (ret == -ETIMEDOUT) { dev_err(qproc->dev, "MBA boot timed out\n"); @@ -1813,14 +1809,10 @@ static int q6v5_init_clocks(struct device *dev, struct clk **clks, for (i = 0; clk_names[i]; i++) { clks[i] = devm_clk_get(dev, clk_names[i]); - if (IS_ERR(clks[i])) { - int rc = PTR_ERR(clks[i]); - - if (rc != -EPROBE_DEFER) - dev_err(dev, "Failed to get %s clock\n", - clk_names[i]); - return rc; - } + if (IS_ERR(clks[i])) + return dev_err_probe(dev, PTR_ERR(clks[i]), + "Failed to get %s clock\n", + clk_names[i]); } return i; @@ -2028,42 +2020,32 @@ static int q6v5_probe(struct platform_device *pdev) ret = q6v5_init_clocks(&pdev->dev, qproc->proxy_clks, desc->proxy_clk_names); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get proxy clocks.\n"); + if (ret < 0) return ret; - } qproc->proxy_clk_count = ret; ret = q6v5_init_clocks(&pdev->dev, qproc->reset_clks, desc->reset_clk_names); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get reset clocks.\n"); + if (ret < 0) return ret; - } qproc->reset_clk_count = ret; ret = q6v5_init_clocks(&pdev->dev, qproc->active_clks, desc->active_clk_names); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get active clocks.\n"); + if (ret < 0) return ret; - } qproc->active_clk_count = ret; ret = q6v5_regulator_init(&pdev->dev, qproc->proxy_regs, desc->proxy_supply); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get proxy regulators.\n"); + if (ret < 0) return ret; - } qproc->proxy_reg_count = ret; ret = q6v5_regulator_init(&pdev->dev, qproc->active_regs, desc->active_supply); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get active regulators.\n"); + if (ret < 0) return ret; - } qproc->active_reg_count = ret; ret = q6v5_pds_attach(&pdev->dev, qproc->proxy_pds, @@ -2073,10 +2055,8 @@ static int q6v5_probe(struct platform_device *pdev) ret = q6v5_regulator_init(&pdev->dev, qproc->fallback_proxy_regs, desc->fallback_proxy_supply); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get fallback proxy regulators.\n"); + if (ret < 0) return ret; - } qproc->fallback_proxy_reg_count = ret; } else if (ret < 0) { dev_err(&pdev->dev, "Failed to init power domains\n"); @@ -2533,7 +2513,7 @@ MODULE_DEVICE_TABLE(of, q6v5_of_match); static struct platform_driver q6v5_driver = { .probe = q6v5_probe, - .remove_new = q6v5_remove, + .remove = q6v5_remove, .driver = { .name = "qcom-q6v5-mss", .of_match_table = q6v5_of_match, diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index ef82835e98a4..97c4bdd9222a 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -453,24 +453,16 @@ static const struct rproc_ops adsp_minidump_ops = { static int adsp_init_clock(struct qcom_adsp *adsp) { - int ret; - adsp->xo = devm_clk_get(adsp->dev, "xo"); - if (IS_ERR(adsp->xo)) { - ret = PTR_ERR(adsp->xo); - if (ret != -EPROBE_DEFER) - dev_err(adsp->dev, "failed to get xo clock"); - return ret; - } + if (IS_ERR(adsp->xo)) + return dev_err_probe(adsp->dev, PTR_ERR(adsp->xo), + "failed to get xo clock"); + adsp->aggre2_clk = devm_clk_get_optional(adsp->dev, "aggre2"); - if (IS_ERR(adsp->aggre2_clk)) { - ret = PTR_ERR(adsp->aggre2_clk); - if (ret != -EPROBE_DEFER) - dev_err(adsp->dev, - "failed to get aggre2 clock"); - return ret; - } + if (IS_ERR(adsp->aggre2_clk)) + return dev_err_probe(adsp->dev, PTR_ERR(adsp->aggre2_clk), + "failed to get aggre2 clock"); return 0; } @@ -716,7 +708,7 @@ static int adsp_probe(struct platform_device *pdev) if (desc->minidump_id) ops = &adsp_minidump_ops; - rproc = devm_rproc_alloc(&pdev->dev, pdev->name, ops, fw_name, sizeof(*adsp)); + rproc = devm_rproc_alloc(&pdev->dev, desc->sysmon_name, ops, fw_name, sizeof(*adsp)); if (!rproc) { dev_err(&pdev->dev, "unable to allocate remoteproc\n"); @@ -759,16 +751,16 @@ static int adsp_probe(struct platform_device *pdev) ret = adsp_init_clock(adsp); if (ret) - goto free_rproc; + goto unassign_mem; ret = adsp_init_regulator(adsp); if (ret) - goto free_rproc; + goto unassign_mem; ret = adsp_pds_attach(&pdev->dev, adsp->proxy_pds, desc->proxy_pd_names); if (ret < 0) - goto free_rproc; + goto unassign_mem; adsp->proxy_pd_count = ret; ret = qcom_q6v5_init(&adsp->q6v5, pdev, rproc, desc->crash_reason_smem, desc->load_state, @@ -784,18 +776,28 @@ static int adsp_probe(struct platform_device *pdev) desc->ssctl_id); if (IS_ERR(adsp->sysmon)) { ret = PTR_ERR(adsp->sysmon); - goto detach_proxy_pds; + goto deinit_remove_pdm_smd_glink; } qcom_add_ssr_subdev(rproc, &adsp->ssr_subdev, desc->ssr_name); ret = rproc_add(rproc); if (ret) - goto detach_proxy_pds; + goto remove_ssr_sysmon; return 0; +remove_ssr_sysmon: + qcom_remove_ssr_subdev(rproc, &adsp->ssr_subdev); + qcom_remove_sysmon_subdev(adsp->sysmon); +deinit_remove_pdm_smd_glink: + qcom_remove_pdm_subdev(rproc, &adsp->pdm_subdev); + qcom_remove_smd_subdev(rproc, &adsp->smd_subdev); + qcom_remove_glink_subdev(rproc, &adsp->glink_subdev); + qcom_q6v5_deinit(&adsp->q6v5); detach_proxy_pds: adsp_pds_detach(adsp, adsp->proxy_pds, adsp->proxy_pd_count); +unassign_mem: + adsp_unassign_memory_region(adsp); free_rproc: device_init_wakeup(adsp->dev, false); @@ -907,6 +909,7 @@ static const struct adsp_data sm8250_adsp_resource = { .crash_reason_smem = 423, .firmware_name = "adsp.mdt", .pas_id = 1, + .minidump_id = 5, .auto_boot = true, .proxy_pd_names = (char*[]){ "lcx", @@ -1124,6 +1127,7 @@ static const struct adsp_data sm8350_cdsp_resource = { .crash_reason_smem = 601, .firmware_name = "cdsp.mdt", .pas_id = 18, + .minidump_id = 7, .auto_boot = true, .proxy_pd_names = (char*[]){ "cx", @@ -1344,7 +1348,7 @@ static const struct adsp_data sc7280_wpss_resource = { .crash_reason_smem = 626, .firmware_name = "wpss.mdt", .pas_id = 6, - .auto_boot = true, + .auto_boot = false, .proxy_pd_names = (char*[]){ "cx", "mx", @@ -1421,6 +1425,7 @@ static const struct of_device_id adsp_of_match[] = { { .compatible = "qcom,sa8775p-cdsp1-pas", .data = &sa8775p_cdsp1_resource}, { .compatible = "qcom,sa8775p-gpdsp0-pas", .data = &sa8775p_gpdsp0_resource}, { .compatible = "qcom,sa8775p-gpdsp1-pas", .data = &sa8775p_gpdsp1_resource}, + { .compatible = "qcom,sar2130p-adsp-pas", .data = &sm8350_adsp_resource}, { .compatible = "qcom,sc7180-adsp-pas", .data = &sm8250_adsp_resource}, { .compatible = "qcom,sc7180-mpss-pas", .data = &mpss_resource_init}, { .compatible = "qcom,sc7280-adsp-pas", .data = &sm8350_adsp_resource}, @@ -1477,7 +1482,7 @@ MODULE_DEVICE_TABLE(of, adsp_of_match); static struct platform_driver adsp_driver = { .probe = adsp_probe, - .remove_new = adsp_remove, + .remove = adsp_remove, .driver = { .name = "qcom_q6v5_pas", .of_match_table = adsp_of_match, diff --git a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c index e913dabae992..93648734a2f2 100644 --- a/drivers/remoteproc/qcom_q6v5_wcss.c +++ b/drivers/remoteproc/qcom_q6v5_wcss.c @@ -902,90 +902,58 @@ static int q6v5_alloc_memory_region(struct q6v5_wcss *wcss) static int q6v5_wcss_init_clock(struct q6v5_wcss *wcss) { - int ret; - wcss->xo = devm_clk_get(wcss->dev, "xo"); - if (IS_ERR(wcss->xo)) { - ret = PTR_ERR(wcss->xo); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get xo clock"); - return ret; - } + if (IS_ERR(wcss->xo)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->xo), + "failed to get xo clock"); wcss->gcc_abhs_cbcr = devm_clk_get(wcss->dev, "gcc_abhs_cbcr"); - if (IS_ERR(wcss->gcc_abhs_cbcr)) { - ret = PTR_ERR(wcss->gcc_abhs_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get gcc abhs clock"); - return ret; - } + if (IS_ERR(wcss->gcc_abhs_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->gcc_abhs_cbcr), + "failed to get gcc abhs clock"); wcss->gcc_axim_cbcr = devm_clk_get(wcss->dev, "gcc_axim_cbcr"); - if (IS_ERR(wcss->gcc_axim_cbcr)) { - ret = PTR_ERR(wcss->gcc_axim_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get gcc axim clock\n"); - return ret; - } + if (IS_ERR(wcss->gcc_axim_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->gcc_axim_cbcr), + "failed to get gcc axim clock\n"); wcss->ahbfabric_cbcr_clk = devm_clk_get(wcss->dev, "lcc_ahbfabric_cbc"); - if (IS_ERR(wcss->ahbfabric_cbcr_clk)) { - ret = PTR_ERR(wcss->ahbfabric_cbcr_clk); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get ahbfabric clock\n"); - return ret; - } + if (IS_ERR(wcss->ahbfabric_cbcr_clk)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->ahbfabric_cbcr_clk), + "failed to get ahbfabric clock\n"); wcss->lcc_csr_cbcr = devm_clk_get(wcss->dev, "tcsr_lcc_cbc"); - if (IS_ERR(wcss->lcc_csr_cbcr)) { - ret = PTR_ERR(wcss->lcc_csr_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get csr cbcr clk\n"); - return ret; - } + if (IS_ERR(wcss->lcc_csr_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->lcc_csr_cbcr), + "failed to get csr cbcr clk\n"); wcss->ahbs_cbcr = devm_clk_get(wcss->dev, "lcc_abhs_cbc"); - if (IS_ERR(wcss->ahbs_cbcr)) { - ret = PTR_ERR(wcss->ahbs_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get ahbs_cbcr clk\n"); - return ret; - } + if (IS_ERR(wcss->ahbs_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->ahbs_cbcr), + "failed to get ahbs_cbcr clk\n"); wcss->tcm_slave_cbcr = devm_clk_get(wcss->dev, "lcc_tcm_slave_cbc"); - if (IS_ERR(wcss->tcm_slave_cbcr)) { - ret = PTR_ERR(wcss->tcm_slave_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get tcm cbcr clk\n"); - return ret; - } + if (IS_ERR(wcss->tcm_slave_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->tcm_slave_cbcr), + "failed to get tcm cbcr clk\n"); wcss->qdsp6ss_abhm_cbcr = devm_clk_get(wcss->dev, "lcc_abhm_cbc"); - if (IS_ERR(wcss->qdsp6ss_abhm_cbcr)) { - ret = PTR_ERR(wcss->qdsp6ss_abhm_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get abhm cbcr clk\n"); - return ret; - } + if (IS_ERR(wcss->qdsp6ss_abhm_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->qdsp6ss_abhm_cbcr), + "failed to get abhm cbcr clk\n"); wcss->qdsp6ss_axim_cbcr = devm_clk_get(wcss->dev, "lcc_axim_cbc"); - if (IS_ERR(wcss->qdsp6ss_axim_cbcr)) { - ret = PTR_ERR(wcss->qdsp6ss_axim_cbcr); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get axim cbcr clk\n"); - return ret; - } + if (IS_ERR(wcss->qdsp6ss_axim_cbcr)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->qdsp6ss_axim_cbcr), + "failed to get axim cbcr clk\n"); wcss->lcc_bcr_sleep = devm_clk_get(wcss->dev, "lcc_bcr_sleep"); - if (IS_ERR(wcss->lcc_bcr_sleep)) { - ret = PTR_ERR(wcss->lcc_bcr_sleep); - if (ret != -EPROBE_DEFER) - dev_err(wcss->dev, "failed to get bcr cbcr clk\n"); - return ret; - } + if (IS_ERR(wcss->lcc_bcr_sleep)) + return dev_err_probe(wcss->dev, PTR_ERR(wcss->lcc_bcr_sleep), + "failed to get bcr cbcr clk\n"); return 0; } @@ -1021,7 +989,6 @@ static int q6v5_wcss_probe(struct platform_device *pdev) wcss = rproc->priv; wcss->dev = &pdev->dev; - wcss->version = desc->version; wcss->version = desc->version; wcss->requires_force_stop = desc->requires_force_stop; @@ -1056,18 +1023,33 @@ static int q6v5_wcss_probe(struct platform_device *pdev) qcom_add_pdm_subdev(rproc, &wcss->pdm_subdev); qcom_add_ssr_subdev(rproc, &wcss->ssr_subdev, "q6wcss"); - if (desc->ssctl_id) + if (desc->ssctl_id) { wcss->sysmon = qcom_add_sysmon_subdev(rproc, desc->sysmon_name, desc->ssctl_id); + if (IS_ERR(wcss->sysmon)) { + ret = PTR_ERR(wcss->sysmon); + goto deinit_remove_subdevs; + } + } ret = rproc_add(rproc); if (ret) - return ret; + goto remove_sysmon_subdev; platform_set_drvdata(pdev, rproc); return 0; + +remove_sysmon_subdev: + if (desc->ssctl_id) + qcom_remove_sysmon_subdev(wcss->sysmon); +deinit_remove_subdevs: + qcom_q6v5_deinit(&wcss->q6v5); + qcom_remove_glink_subdev(rproc, &wcss->glink_subdev); + qcom_remove_pdm_subdev(rproc, &wcss->pdm_subdev); + qcom_remove_ssr_subdev(rproc, &wcss->ssr_subdev); + return ret; } static void q6v5_wcss_remove(struct platform_device *pdev) @@ -1111,7 +1093,7 @@ MODULE_DEVICE_TABLE(of, q6v5_wcss_of_match); static struct platform_driver q6v5_wcss_driver = { .probe = q6v5_wcss_probe, - .remove_new = q6v5_wcss_remove, + .remove = q6v5_wcss_remove, .driver = { .name = "qcom-q6v5-wcss-pil", .of_match_table = q6v5_wcss_of_match, diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index a7bb9da27029..5b5664603eed 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -682,7 +682,7 @@ MODULE_DEVICE_TABLE(of, wcnss_of_match); static struct platform_driver wcnss_driver = { .probe = wcnss_probe, - .remove_new = wcnss_remove, + .remove = wcnss_remove, .driver = { .name = "qcom-wcnss-pil", .of_match_table = wcnss_of_match, diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c index dd36fd077911..b989718776bd 100644 --- a/drivers/remoteproc/qcom_wcnss_iris.c +++ b/drivers/remoteproc/qcom_wcnss_iris.c @@ -155,9 +155,8 @@ struct qcom_iris *qcom_iris_probe(struct device *parent, bool *use_48mhz_xo) iris->xo_clk = devm_clk_get(&iris->dev, "xo"); if (IS_ERR(iris->xo_clk)) { - ret = PTR_ERR(iris->xo_clk); - if (ret != -EPROBE_DEFER) - dev_err(&iris->dev, "failed to acquire xo clk\n"); + ret = dev_err_probe(&iris->dev, PTR_ERR(iris->xo_clk), + "failed to acquire xo clk\n"); goto err_device_del; } diff --git a/drivers/remoteproc/rcar_rproc.c b/drivers/remoteproc/rcar_rproc.c index cc17e8421f65..921d853594f4 100644 --- a/drivers/remoteproc/rcar_rproc.c +++ b/drivers/remoteproc/rcar_rproc.c @@ -214,7 +214,7 @@ MODULE_DEVICE_TABLE(of, rcar_rproc_of_match); static struct platform_driver rcar_rproc_driver = { .probe = rcar_rproc_probe, - .remove_new = rcar_rproc_remove, + .remove = rcar_rproc_remove, .driver = { .name = "rcar-rproc", .of_match_table = rcar_rproc_of_match, diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index d3f39009b28e..25a655f33ec0 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -593,7 +593,7 @@ static void rproc_virtio_remove(struct platform_device *pdev) /* Platform driver */ static struct platform_driver rproc_virtio_driver = { .probe = rproc_virtio_probe, - .remove_new = rproc_virtio_remove, + .remove = rproc_virtio_remove, .driver = { .name = "rproc-virtio", }, diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c index 1340be9d0110..5df99bae7131 100644 --- a/drivers/remoteproc/st_remoteproc.c +++ b/drivers/remoteproc/st_remoteproc.c @@ -457,7 +457,7 @@ static void st_rproc_remove(struct platform_device *pdev) static struct platform_driver st_rproc_driver = { .probe = st_rproc_probe, - .remove_new = st_rproc_remove, + .remove = st_rproc_remove, .driver = { .name = "st-rproc", .of_match_table = of_match_ptr(st_rproc_match), diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c index 8c7f7950b80e..b02b36a3f515 100644 --- a/drivers/remoteproc/stm32_rproc.c +++ b/drivers/remoteproc/stm32_rproc.c @@ -946,7 +946,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(stm32_rproc_pm_ops, static struct platform_driver stm32_rproc_driver = { .probe = stm32_rproc_probe, - .remove_new = stm32_rproc_remove, + .remove = stm32_rproc_remove, .driver = { .name = "stm32-rproc", .pm = pm_ptr(&stm32_rproc_pm_ops), diff --git a/drivers/remoteproc/ti_k3_dsp_remoteproc.c b/drivers/remoteproc/ti_k3_dsp_remoteproc.c index 8be3f631c192..a695890254ff 100644 --- a/drivers/remoteproc/ti_k3_dsp_remoteproc.c +++ b/drivers/remoteproc/ti_k3_dsp_remoteproc.c @@ -403,7 +403,7 @@ static struct resource_table *k3_dsp_get_loaded_rsc_table(struct rproc *rproc, * the hard-coded value suffices to support the IPC-only mode. */ *rsc_table_sz = 256; - return (struct resource_table *)kproc->rmem[0].cpu_addr; + return (__force struct resource_table *)kproc->rmem[0].cpu_addr; } /* @@ -576,11 +576,9 @@ static int k3_dsp_reserved_mem_init(struct k3_dsp_rproc *kproc) return -EINVAL; rmem = of_reserved_mem_lookup(rmem_np); - if (!rmem) { - of_node_put(rmem_np); - return -EINVAL; - } of_node_put(rmem_np); + if (!rmem) + return -EINVAL; kproc->rmem[i].bus_addr = rmem->base; /* 64-bit address regions currently not supported */ @@ -793,7 +791,7 @@ MODULE_DEVICE_TABLE(of, k3_dsp_of_match); static struct platform_driver k3_dsp_rproc_driver = { .probe = k3_dsp_rproc_probe, - .remove_new = k3_dsp_rproc_remove, + .remove = k3_dsp_rproc_remove, .driver = { .name = "k3-dsp-rproc", .of_match_table = k3_dsp_of_match, diff --git a/drivers/remoteproc/ti_k3_m4_remoteproc.c b/drivers/remoteproc/ti_k3_m4_remoteproc.c index 09f0484a90e1..a16fb165fced 100644 --- a/drivers/remoteproc/ti_k3_m4_remoteproc.c +++ b/drivers/remoteproc/ti_k3_m4_remoteproc.c @@ -433,11 +433,9 @@ static int k3_m4_reserved_mem_init(struct k3_m4_rproc *kproc) return -EINVAL; rmem = of_reserved_mem_lookup(rmem_np); - if (!rmem) { - of_node_put(rmem_np); - return -EINVAL; - } of_node_put(rmem_np); + if (!rmem) + return -EINVAL; kproc->rmem[i].bus_addr = rmem->base; /* 64-bit address regions currently not supported */ diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c index 747ee467da88..6560b7954027 100644 --- a/drivers/remoteproc/ti_k3_r5_remoteproc.c +++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c @@ -487,10 +487,10 @@ static int k3_r5_rproc_prepare(struct rproc *rproc) * can be effective on all TCM addresses. */ dev_dbg(dev, "zeroing out ATCM memory\n"); - memset(core->mem[0].cpu_addr, 0x00, core->mem[0].size); + memset_io(core->mem[0].cpu_addr, 0x00, core->mem[0].size); dev_dbg(dev, "zeroing out BTCM memory\n"); - memset(core->mem[1].cpu_addr, 0x00, core->mem[1].size); + memset_io(core->mem[1].cpu_addr, 0x00, core->mem[1].size); return 0; } @@ -717,7 +717,7 @@ static struct resource_table *k3_r5_get_loaded_rsc_table(struct rproc *rproc, * the hard-coded value suffices to support the IPC-only mode. */ *rsc_table_sz = 256; - return (struct resource_table *)kproc->rmem[0].cpu_addr; + return (__force struct resource_table *)kproc->rmem[0].cpu_addr; } /* @@ -1001,12 +1001,11 @@ static int k3_r5_reserved_mem_init(struct k3_r5_rproc *kproc) } rmem = of_reserved_mem_lookup(rmem_np); + of_node_put(rmem_np); if (!rmem) { - of_node_put(rmem_np); ret = -EINVAL; goto unmap_rmem; } - of_node_put(rmem_np); kproc->rmem[i].bus_addr = rmem->base; /* @@ -1558,11 +1557,7 @@ static int k3_r5_core_of_init(struct platform_device *pdev) core->ti_sci = devm_ti_sci_get_by_phandle(dev, "ti,sci"); if (IS_ERR(core->ti_sci)) { - ret = PTR_ERR(core->ti_sci); - if (ret != -EPROBE_DEFER) { - dev_err(dev, "failed to get ti-sci handle, ret = %d\n", - ret); - } + ret = dev_err_probe(dev, PTR_ERR(core->ti_sci), "failed to get ti-sci handle\n"); core->ti_sci = NULL; goto err; } @@ -1578,18 +1573,14 @@ static int k3_r5_core_of_init(struct platform_device *pdev) ret = PTR_ERR_OR_ZERO(core->reset); if (!ret) ret = -ENODEV; - if (ret != -EPROBE_DEFER) { - dev_err(dev, "failed to get reset handle, ret = %d\n", - ret); - } + dev_err_probe(dev, ret, "failed to get reset handle\n"); goto err; } core->tsp = ti_sci_proc_of_get_tsp(dev, core->ti_sci); if (IS_ERR(core->tsp)) { - ret = PTR_ERR(core->tsp); - dev_err(dev, "failed to construct ti-sci proc control, ret = %d\n", - ret); + ret = dev_err_probe(dev, PTR_ERR(core->tsp), + "failed to construct ti-sci proc control\n"); goto err; } @@ -1659,16 +1650,14 @@ static int k3_r5_cluster_of_init(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev_of_node(dev); struct platform_device *cpdev; - struct device_node *child; struct k3_r5_core *core; int ret; - for_each_available_child_of_node(np, child) { + for_each_available_child_of_node_scoped(np, child) { cpdev = of_find_device_by_node(child); if (!cpdev) { ret = -ENODEV; dev_err(dev, "could not get R5 core platform device\n"); - of_node_put(child); goto fail; } @@ -1677,7 +1666,6 @@ static int k3_r5_cluster_of_init(struct platform_device *pdev) dev_err(dev, "k3_r5_core_of_init failed, ret = %d\n", ret); put_device(&cpdev->dev); - of_node_put(child); goto fail; } @@ -1718,11 +1706,8 @@ static int k3_r5_probe(struct platform_device *pdev) init_waitqueue_head(&cluster->core_transition); ret = of_property_read_u32(np, "ti,cluster-mode", &cluster->mode); - if (ret < 0 && ret != -EINVAL) { - dev_err(dev, "invalid format for ti,cluster-mode, ret = %d\n", - ret); - return ret; - } + if (ret < 0 && ret != -EINVAL) + return dev_err_probe(dev, ret, "invalid format for ti,cluster-mode\n"); if (ret == -EINVAL) { /* @@ -1741,49 +1726,39 @@ static int k3_r5_probe(struct platform_device *pdev) } if ((cluster->mode == CLUSTER_MODE_SINGLECPU && !data->single_cpu_mode) || - (cluster->mode == CLUSTER_MODE_SINGLECORE && !data->is_single_core)) { - dev_err(dev, "Cluster mode = %d is not supported on this SoC\n", cluster->mode); - return -EINVAL; - } + (cluster->mode == CLUSTER_MODE_SINGLECORE && !data->is_single_core)) + return dev_err_probe(dev, -EINVAL, + "Cluster mode = %d is not supported on this SoC\n", + cluster->mode); num_cores = of_get_available_child_count(np); - if (num_cores != 2 && !data->is_single_core) { - dev_err(dev, "MCU cluster requires both R5F cores to be enabled but num_cores is set to = %d\n", - num_cores); - return -ENODEV; - } + if (num_cores != 2 && !data->is_single_core) + return dev_err_probe(dev, -ENODEV, + "MCU cluster requires both R5F cores to be enabled but num_cores is set to = %d\n", + num_cores); - if (num_cores != 1 && data->is_single_core) { - dev_err(dev, "SoC supports only single core R5 but num_cores is set to %d\n", - num_cores); - return -ENODEV; - } + if (num_cores != 1 && data->is_single_core) + return dev_err_probe(dev, -ENODEV, + "SoC supports only single core R5 but num_cores is set to %d\n", + num_cores); platform_set_drvdata(pdev, cluster); ret = devm_of_platform_populate(dev); - if (ret) { - dev_err(dev, "devm_of_platform_populate failed, ret = %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "devm_of_platform_populate failed\n"); ret = k3_r5_cluster_of_init(pdev); - if (ret) { - dev_err(dev, "k3_r5_cluster_of_init failed, ret = %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "k3_r5_cluster_of_init failed\n"); ret = devm_add_action_or_reset(dev, k3_r5_cluster_of_exit, pdev); if (ret) return ret; ret = k3_r5_cluster_rproc_init(pdev); - if (ret) { - dev_err(dev, "k3_r5_cluster_rproc_init failed, ret = %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "k3_r5_cluster_rproc_init failed\n"); ret = devm_add_action_or_reset(dev, k3_r5_cluster_rproc_exit, pdev); if (ret) diff --git a/drivers/remoteproc/wkup_m3_rproc.c b/drivers/remoteproc/wkup_m3_rproc.c index 36a55f7ffa64..d8be21e71721 100644 --- a/drivers/remoteproc/wkup_m3_rproc.c +++ b/drivers/remoteproc/wkup_m3_rproc.c @@ -251,7 +251,7 @@ static const struct dev_pm_ops wkup_m3_rproc_pm_ops = { static struct platform_driver wkup_m3_rproc_driver = { .probe = wkup_m3_rproc_probe, - .remove_new = wkup_m3_rproc_remove, + .remove = wkup_m3_rproc_remove, .driver = { .name = "wkup_m3_rproc", .of_match_table = wkup_m3_rproc_of_match, diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index d3af1dfa3c7d..a2f9d85c7156 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1204,7 +1204,8 @@ void qcom_glink_native_rx(struct qcom_glink *glink) ret = qcom_glink_rx_open_ack(glink, param1); break; case GLINK_CMD_OPEN: - ret = qcom_glink_rx_defer(glink, param2); + /* upper 16 bits of param2 are the "prio" field */ + ret = qcom_glink_rx_defer(glink, param2 & 0xffff); break; case GLINK_CMD_TX_DATA: case GLINK_CMD_TX_DATA_CONT: diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 61e7ae524b1f..d3f9686e26e7 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -220,6 +220,15 @@ config DEVFREQ_THERMAL If you want this support, you should say Y here. +config PCIE_THERMAL + bool "PCIe cooling support" + depends on PCIEPORTBUS + help + This implements PCIe cooling mechanism through bandwidth reduction + for PCIe devices. + + If you want this support, you should say Y here. + config THERMAL_EMULATION bool "Thermal emulation mode support" help diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index 1e1559bb971e..9abf43a74f2b 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -32,6 +32,8 @@ thermal_sys-$(CONFIG_CPU_IDLE_THERMAL) += cpuidle_cooling.o # devfreq cooling thermal_sys-$(CONFIG_DEVFREQ_THERMAL) += devfreq_cooling.o +thermal_sys-$(CONFIG_PCIE_THERMAL) += pcie_cooling.o + obj-$(CONFIG_K3_THERMAL) += k3_bandgap.o k3_j72xx_bandgap.o # platform thermal drivers obj-y += broadcom/ diff --git a/drivers/thermal/pcie_cooling.c b/drivers/thermal/pcie_cooling.c new file mode 100644 index 000000000000..a876d64f1582 --- /dev/null +++ b/drivers/thermal/pcie_cooling.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PCIe cooling device + * + * Copyright (C) 2023-2024 Intel Corporation + */ + +#include <linux/build_bug.h> +#include <linux/cleanup.h> +#include <linux/err.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pci-bwctrl.h> +#include <linux/slab.h> +#include <linux/sprintf.h> +#include <linux/thermal.h> + +#define COOLING_DEV_TYPE_PREFIX "PCIe_Port_Link_Speed_" + +static int pcie_cooling_get_max_level(struct thermal_cooling_device *cdev, unsigned long *state) +{ + struct pci_dev *port = cdev->devdata; + + /* cooling state 0 is same as the maximum PCIe speed */ + *state = port->subordinate->max_bus_speed - PCIE_SPEED_2_5GT; + + return 0; +} + +static int pcie_cooling_get_cur_level(struct thermal_cooling_device *cdev, unsigned long *state) +{ + struct pci_dev *port = cdev->devdata; + + /* cooling state 0 is same as the maximum PCIe speed */ + *state = cdev->max_state - (port->subordinate->cur_bus_speed - PCIE_SPEED_2_5GT); + + return 0; +} + +static int pcie_cooling_set_cur_level(struct thermal_cooling_device *cdev, unsigned long state) +{ + struct pci_dev *port = cdev->devdata; + enum pci_bus_speed speed; + + /* cooling state 0 is same as the maximum PCIe speed */ + speed = (cdev->max_state - state) + PCIE_SPEED_2_5GT; + + return pcie_set_target_speed(port, speed, true); +} + +static struct thermal_cooling_device_ops pcie_cooling_ops = { + .get_max_state = pcie_cooling_get_max_level, + .get_cur_state = pcie_cooling_get_cur_level, + .set_cur_state = pcie_cooling_set_cur_level, +}; + +struct thermal_cooling_device *pcie_cooling_device_register(struct pci_dev *port) +{ + char *name __free(kfree) = + kasprintf(GFP_KERNEL, COOLING_DEV_TYPE_PREFIX "%s", pci_name(port)); + if (!name) + return ERR_PTR(-ENOMEM); + + return thermal_cooling_device_register(name, port, &pcie_cooling_ops); +} + +void pcie_cooling_device_unregister(struct thermal_cooling_device *cdev) +{ + thermal_cooling_device_unregister(cdev); +} + +/* For bus_speed <-> state arithmetic */ +static_assert(PCIE_SPEED_2_5GT + 1 == PCIE_SPEED_5_0GT); +static_assert(PCIE_SPEED_5_0GT + 1 == PCIE_SPEED_8_0GT); +static_assert(PCIE_SPEED_8_0GT + 1 == PCIE_SPEED_16_0GT); +static_assert(PCIE_SPEED_16_0GT + 1 == PCIE_SPEED_32_0GT); +static_assert(PCIE_SPEED_32_0GT + 1 == PCIE_SPEED_64_0GT); + +MODULE_AUTHOR("Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>"); +MODULE_DESCRIPTION("PCIe cooling driver"); diff --git a/drivers/tty/serial/rp2.c b/drivers/tty/serial/rp2.c index 8bab2aedc499..6d99a02dd439 100644 --- a/drivers/tty/serial/rp2.c +++ b/drivers/tty/serial/rp2.c @@ -698,7 +698,6 @@ static int rp2_probe(struct pci_dev *pdev, const struct firmware *fw; struct rp2_card *card; struct rp2_uart_port *ports; - void __iomem * const *bars; int rc; card = devm_kzalloc(&pdev->dev, sizeof(*card), GFP_KERNEL); @@ -711,13 +710,16 @@ static int rp2_probe(struct pci_dev *pdev, if (rc) return rc; - rc = pcim_iomap_regions_request_all(pdev, 0x03, DRV_NAME); + rc = pcim_request_all_regions(pdev, DRV_NAME); if (rc) return rc; - bars = pcim_iomap_table(pdev); - card->bar0 = bars[0]; - card->bar1 = bars[1]; + card->bar0 = pcim_iomap(pdev, 0, 0); + if (!card->bar0) + return -ENOMEM; + card->bar1 = pcim_iomap(pdev, 1, 0); + if (!card->bar1) + return -ENOMEM; card->pdev = pdev; rp2_decode_cap(id, &card->n_ports, &card->smpte); diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index 577c56302314..8dd1d7189c3b 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -15,10 +15,10 @@ #include <linux/module.h> #include <linux/namei.h> #include <linux/skbuff.h> -#include <linux/mount.h> #include <linux/pagemap.h> #include <linux/key.h> -#include <linux/parser.h> +#include <linux/fs_context.h> +#include <linux/fs_parser.h> #include <linux/fs_stack.h> #include <linux/slab.h> #include <linux/magic.h> @@ -153,32 +153,30 @@ void ecryptfs_put_lower_file(struct inode *inode) } } -enum { ecryptfs_opt_sig, ecryptfs_opt_ecryptfs_sig, - ecryptfs_opt_cipher, ecryptfs_opt_ecryptfs_cipher, - ecryptfs_opt_ecryptfs_key_bytes, - ecryptfs_opt_passthrough, ecryptfs_opt_xattr_metadata, - ecryptfs_opt_encrypted_view, ecryptfs_opt_fnek_sig, - ecryptfs_opt_fn_cipher, ecryptfs_opt_fn_cipher_key_bytes, - ecryptfs_opt_unlink_sigs, ecryptfs_opt_mount_auth_tok_only, - ecryptfs_opt_check_dev_ruid, - ecryptfs_opt_err }; - -static const match_table_t tokens = { - {ecryptfs_opt_sig, "sig=%s"}, - {ecryptfs_opt_ecryptfs_sig, "ecryptfs_sig=%s"}, - {ecryptfs_opt_cipher, "cipher=%s"}, - {ecryptfs_opt_ecryptfs_cipher, "ecryptfs_cipher=%s"}, - {ecryptfs_opt_ecryptfs_key_bytes, "ecryptfs_key_bytes=%u"}, - {ecryptfs_opt_passthrough, "ecryptfs_passthrough"}, - {ecryptfs_opt_xattr_metadata, "ecryptfs_xattr_metadata"}, - {ecryptfs_opt_encrypted_view, "ecryptfs_encrypted_view"}, - {ecryptfs_opt_fnek_sig, "ecryptfs_fnek_sig=%s"}, - {ecryptfs_opt_fn_cipher, "ecryptfs_fn_cipher=%s"}, - {ecryptfs_opt_fn_cipher_key_bytes, "ecryptfs_fn_key_bytes=%u"}, - {ecryptfs_opt_unlink_sigs, "ecryptfs_unlink_sigs"}, - {ecryptfs_opt_mount_auth_tok_only, "ecryptfs_mount_auth_tok_only"}, - {ecryptfs_opt_check_dev_ruid, "ecryptfs_check_dev_ruid"}, - {ecryptfs_opt_err, NULL} +enum { + Opt_sig, Opt_ecryptfs_sig, Opt_cipher, Opt_ecryptfs_cipher, + Opt_ecryptfs_key_bytes, Opt_passthrough, Opt_xattr_metadata, + Opt_encrypted_view, Opt_fnek_sig, Opt_fn_cipher, + Opt_fn_cipher_key_bytes, Opt_unlink_sigs, Opt_mount_auth_tok_only, + Opt_check_dev_ruid +}; + +static const struct fs_parameter_spec ecryptfs_fs_param_spec[] = { + fsparam_string ("sig", Opt_sig), + fsparam_string ("ecryptfs_sig", Opt_ecryptfs_sig), + fsparam_string ("cipher", Opt_cipher), + fsparam_string ("ecryptfs_cipher", Opt_ecryptfs_cipher), + fsparam_u32 ("ecryptfs_key_bytes", Opt_ecryptfs_key_bytes), + fsparam_flag ("ecryptfs_passthrough", Opt_passthrough), + fsparam_flag ("ecryptfs_xattr_metadata", Opt_xattr_metadata), + fsparam_flag ("ecryptfs_encrypted_view", Opt_encrypted_view), + fsparam_string ("ecryptfs_fnek_sig", Opt_fnek_sig), + fsparam_string ("ecryptfs_fn_cipher", Opt_fn_cipher), + fsparam_u32 ("ecryptfs_fn_key_bytes", Opt_fn_cipher_key_bytes), + fsparam_flag ("ecryptfs_unlink_sigs", Opt_unlink_sigs), + fsparam_flag ("ecryptfs_mount_auth_tok_only", Opt_mount_auth_tok_only), + fsparam_flag ("ecryptfs_check_dev_ruid", Opt_check_dev_ruid), + {} }; static int ecryptfs_init_global_auth_toks( @@ -219,19 +217,20 @@ static void ecryptfs_init_mount_crypt_stat( mount_crypt_stat->flags |= ECRYPTFS_MOUNT_CRYPT_STAT_INITIALIZED; } +struct ecryptfs_fs_context { + /* Mount option status trackers */ + bool check_ruid; + bool sig_set; + bool cipher_name_set; + bool cipher_key_bytes_set; + bool fn_cipher_name_set; + bool fn_cipher_key_bytes_set; +}; + /** - * ecryptfs_parse_options - * @sbi: The ecryptfs super block - * @options: The options passed to the kernel - * @check_ruid: set to 1 if device uid should be checked against the ruid - * - * Parse mount options: - * debug=N - ecryptfs_verbosity level for debug output - * sig=XXX - description(signature) of the key to use - * - * Returns the dentry object of the lower-level (lower/interposed) - * directory; We want to mount our stackable file system on top of - * that lower directory. + * ecryptfs_parse_param + * @fc: The ecryptfs filesystem context + * @param: The mount parameter to parse * * The signature of the key to use must be the description of a key * already in the keyring. Mounting will fail if the key can not be @@ -239,143 +238,118 @@ static void ecryptfs_init_mount_crypt_stat( * * Returns zero on success; non-zero on error */ -static int ecryptfs_parse_options(struct ecryptfs_sb_info *sbi, char *options, - uid_t *check_ruid) +static int ecryptfs_parse_param( + struct fs_context *fc, + struct fs_parameter *param) { - char *p; - int rc = 0; - int sig_set = 0; - int cipher_name_set = 0; - int fn_cipher_name_set = 0; - int cipher_key_bytes; - int cipher_key_bytes_set = 0; - int fn_cipher_key_bytes; - int fn_cipher_key_bytes_set = 0; + int rc; + int opt; + struct fs_parse_result result; + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; struct ecryptfs_mount_crypt_stat *mount_crypt_stat = &sbi->mount_crypt_stat; - substring_t args[MAX_OPT_ARGS]; - int token; - char *sig_src; - char *cipher_name_src; - char *fn_cipher_name_src; - char *fnek_src; - char *cipher_key_bytes_src; - char *fn_cipher_key_bytes_src; - u8 cipher_code; - *check_ruid = 0; + opt = fs_parse(fc, ecryptfs_fs_param_spec, param, &result); + if (opt < 0) + return opt; - if (!options) { - rc = -EINVAL; - goto out; - } - ecryptfs_init_mount_crypt_stat(mount_crypt_stat); - while ((p = strsep(&options, ",")) != NULL) { - if (!*p) - continue; - token = match_token(p, tokens, args); - switch (token) { - case ecryptfs_opt_sig: - case ecryptfs_opt_ecryptfs_sig: - sig_src = args[0].from; - rc = ecryptfs_add_global_auth_tok(mount_crypt_stat, - sig_src, 0); - if (rc) { - printk(KERN_ERR "Error attempting to register " - "global sig; rc = [%d]\n", rc); - goto out; - } - sig_set = 1; - break; - case ecryptfs_opt_cipher: - case ecryptfs_opt_ecryptfs_cipher: - cipher_name_src = args[0].from; - strscpy(mount_crypt_stat->global_default_cipher_name, - cipher_name_src); - cipher_name_set = 1; - break; - case ecryptfs_opt_ecryptfs_key_bytes: - cipher_key_bytes_src = args[0].from; - cipher_key_bytes = - (int)simple_strtol(cipher_key_bytes_src, - &cipher_key_bytes_src, 0); - mount_crypt_stat->global_default_cipher_key_size = - cipher_key_bytes; - cipher_key_bytes_set = 1; - break; - case ecryptfs_opt_passthrough: - mount_crypt_stat->flags |= - ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED; - break; - case ecryptfs_opt_xattr_metadata: - mount_crypt_stat->flags |= - ECRYPTFS_XATTR_METADATA_ENABLED; - break; - case ecryptfs_opt_encrypted_view: - mount_crypt_stat->flags |= - ECRYPTFS_XATTR_METADATA_ENABLED; - mount_crypt_stat->flags |= - ECRYPTFS_ENCRYPTED_VIEW_ENABLED; - break; - case ecryptfs_opt_fnek_sig: - fnek_src = args[0].from; - strscpy(mount_crypt_stat->global_default_fnek_sig, - fnek_src); - rc = ecryptfs_add_global_auth_tok( - mount_crypt_stat, - mount_crypt_stat->global_default_fnek_sig, - ECRYPTFS_AUTH_TOK_FNEK); - if (rc) { - printk(KERN_ERR "Error attempting to register " - "global fnek sig [%s]; rc = [%d]\n", - mount_crypt_stat->global_default_fnek_sig, - rc); - goto out; - } - mount_crypt_stat->flags |= - (ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES - | ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK); - break; - case ecryptfs_opt_fn_cipher: - fn_cipher_name_src = args[0].from; - strscpy(mount_crypt_stat->global_default_fn_cipher_name, - fn_cipher_name_src); - fn_cipher_name_set = 1; - break; - case ecryptfs_opt_fn_cipher_key_bytes: - fn_cipher_key_bytes_src = args[0].from; - fn_cipher_key_bytes = - (int)simple_strtol(fn_cipher_key_bytes_src, - &fn_cipher_key_bytes_src, 0); - mount_crypt_stat->global_default_fn_cipher_key_bytes = - fn_cipher_key_bytes; - fn_cipher_key_bytes_set = 1; - break; - case ecryptfs_opt_unlink_sigs: - mount_crypt_stat->flags |= ECRYPTFS_UNLINK_SIGS; - break; - case ecryptfs_opt_mount_auth_tok_only: - mount_crypt_stat->flags |= - ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY; - break; - case ecryptfs_opt_check_dev_ruid: - *check_ruid = 1; - break; - case ecryptfs_opt_err: - default: - printk(KERN_WARNING - "%s: eCryptfs: unrecognized option [%s]\n", - __func__, p); + switch (opt) { + case Opt_sig: + case Opt_ecryptfs_sig: + rc = ecryptfs_add_global_auth_tok(mount_crypt_stat, + param->string, 0); + if (rc) { + printk(KERN_ERR "Error attempting to register " + "global sig; rc = [%d]\n", rc); + return rc; + } + ctx->sig_set = 1; + break; + case Opt_cipher: + case Opt_ecryptfs_cipher: + strscpy(mount_crypt_stat->global_default_cipher_name, + param->string); + ctx->cipher_name_set = 1; + break; + case Opt_ecryptfs_key_bytes: + mount_crypt_stat->global_default_cipher_key_size = + result.uint_32; + ctx->cipher_key_bytes_set = 1; + break; + case Opt_passthrough: + mount_crypt_stat->flags |= + ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED; + break; + case Opt_xattr_metadata: + mount_crypt_stat->flags |= ECRYPTFS_XATTR_METADATA_ENABLED; + break; + case Opt_encrypted_view: + mount_crypt_stat->flags |= ECRYPTFS_XATTR_METADATA_ENABLED; + mount_crypt_stat->flags |= ECRYPTFS_ENCRYPTED_VIEW_ENABLED; + break; + case Opt_fnek_sig: + strscpy(mount_crypt_stat->global_default_fnek_sig, + param->string); + rc = ecryptfs_add_global_auth_tok( + mount_crypt_stat, + mount_crypt_stat->global_default_fnek_sig, + ECRYPTFS_AUTH_TOK_FNEK); + if (rc) { + printk(KERN_ERR "Error attempting to register " + "global fnek sig [%s]; rc = [%d]\n", + mount_crypt_stat->global_default_fnek_sig, rc); + return rc; } + mount_crypt_stat->flags |= + (ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES + | ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK); + break; + case Opt_fn_cipher: + strscpy(mount_crypt_stat->global_default_fn_cipher_name, + param->string); + ctx->fn_cipher_name_set = 1; + break; + case Opt_fn_cipher_key_bytes: + mount_crypt_stat->global_default_fn_cipher_key_bytes = + result.uint_32; + ctx->fn_cipher_key_bytes_set = 1; + break; + case Opt_unlink_sigs: + mount_crypt_stat->flags |= ECRYPTFS_UNLINK_SIGS; + break; + case Opt_mount_auth_tok_only: + mount_crypt_stat->flags |= ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY; + break; + case Opt_check_dev_ruid: + ctx->check_ruid = 1; + break; + default: + return -EINVAL; } - if (!sig_set) { + + return 0; +} + +static int ecryptfs_validate_options(struct fs_context *fc) +{ + int rc = 0; + u8 cipher_code; + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + + + mount_crypt_stat = &sbi->mount_crypt_stat; + + if (!ctx->sig_set) { rc = -EINVAL; ecryptfs_printk(KERN_ERR, "You must supply at least one valid " "auth tok signature as a mount " "parameter; see the eCryptfs README\n"); goto out; } - if (!cipher_name_set) { + if (!ctx->cipher_name_set) { int cipher_name_len = strlen(ECRYPTFS_DEFAULT_CIPHER); BUG_ON(cipher_name_len > ECRYPTFS_MAX_CIPHER_NAME_SIZE); @@ -383,13 +357,13 @@ static int ecryptfs_parse_options(struct ecryptfs_sb_info *sbi, char *options, ECRYPTFS_DEFAULT_CIPHER); } if ((mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) - && !fn_cipher_name_set) + && !ctx->fn_cipher_name_set) strcpy(mount_crypt_stat->global_default_fn_cipher_name, mount_crypt_stat->global_default_cipher_name); - if (!cipher_key_bytes_set) + if (!ctx->cipher_key_bytes_set) mount_crypt_stat->global_default_cipher_key_size = 0; if ((mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) - && !fn_cipher_key_bytes_set) + && !ctx->fn_cipher_key_bytes_set) mount_crypt_stat->global_default_fn_cipher_key_bytes = mount_crypt_stat->global_default_cipher_key_size; @@ -453,45 +427,35 @@ struct kmem_cache *ecryptfs_sb_info_cache; static struct file_system_type ecryptfs_fs_type; /* - * ecryptfs_mount - * @fs_type: The filesystem type that the superblock should belong to - * @flags: The flags associated with the mount - * @dev_name: The path to mount over - * @raw_data: The options passed into the kernel + * ecryptfs_get_tree + * @fc: The filesystem context */ -static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags, - const char *dev_name, void *raw_data) +static int ecryptfs_get_tree(struct fs_context *fc) { struct super_block *s; - struct ecryptfs_sb_info *sbi; + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; struct ecryptfs_mount_crypt_stat *mount_crypt_stat; struct ecryptfs_dentry_info *root_info; const char *err = "Getting sb failed"; struct inode *inode; struct path path; - uid_t check_ruid; int rc; - sbi = kmem_cache_zalloc(ecryptfs_sb_info_cache, GFP_KERNEL); - if (!sbi) { - rc = -ENOMEM; - goto out; - } - - if (!dev_name) { + if (!fc->source) { rc = -EINVAL; err = "Device name cannot be null"; goto out; } - rc = ecryptfs_parse_options(sbi, raw_data, &check_ruid); + mount_crypt_stat = &sbi->mount_crypt_stat; + rc = ecryptfs_validate_options(fc); if (rc) { - err = "Error parsing options"; + err = "Error validating options"; goto out; } - mount_crypt_stat = &sbi->mount_crypt_stat; - s = sget(fs_type, NULL, set_anon_super, flags, NULL); + s = sget_fc(fc, NULL, set_anon_super_fc); if (IS_ERR(s)) { rc = PTR_ERR(s); goto out; @@ -510,7 +474,7 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags s->s_d_op = &ecryptfs_dops; err = "Reading sb failed"; - rc = kern_path(dev_name, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + rc = kern_path(fc->source, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); if (rc) { ecryptfs_printk(KERN_WARNING, "kern_path() failed\n"); goto out1; @@ -529,7 +493,8 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags goto out_free; } - if (check_ruid && !uid_eq(d_inode(path.dentry)->i_uid, current_uid())) { + if (ctx->check_ruid && + !uid_eq(d_inode(path.dentry)->i_uid, current_uid())) { rc = -EPERM; printk(KERN_ERR "Mount of device (uid: %d) not owned by " "requested user (uid: %d)\n", @@ -544,7 +509,7 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags * Set the POSIX ACL flag based on whether they're enabled in the lower * mount. */ - s->s_flags = flags & ~SB_POSIXACL; + s->s_flags = fc->sb_flags & ~SB_POSIXACL; s->s_flags |= path.dentry->d_sb->s_flags & SB_POSIXACL; /** @@ -587,19 +552,19 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags root_info->lower_path = path; s->s_flags |= SB_ACTIVE; - return dget(s->s_root); + fc->root = dget(s->s_root); + return 0; out_free: path_put(&path); out1: deactivate_locked_super(s); out: - if (sbi) { + if (sbi) ecryptfs_destroy_mount_crypt_stat(&sbi->mount_crypt_stat); - kmem_cache_free(ecryptfs_sb_info_cache, sbi); - } + printk(KERN_ERR "%s; rc = [%d]\n", err, rc); - return ERR_PTR(rc); + return rc; } /** @@ -618,10 +583,54 @@ static void ecryptfs_kill_block_super(struct super_block *sb) kmem_cache_free(ecryptfs_sb_info_cache, sb_info); } +static void ecryptfs_free_fc(struct fs_context *fc) +{ + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; + + kfree(ctx); + + if (sbi) { + ecryptfs_destroy_mount_crypt_stat(&sbi->mount_crypt_stat); + kmem_cache_free(ecryptfs_sb_info_cache, sbi); + } +} + +static const struct fs_context_operations ecryptfs_context_ops = { + .free = ecryptfs_free_fc, + .parse_param = ecryptfs_parse_param, + .get_tree = ecryptfs_get_tree, + .reconfigure = NULL, +}; + +static int ecryptfs_init_fs_context(struct fs_context *fc) +{ + struct ecryptfs_fs_context *ctx; + struct ecryptfs_sb_info *sbi = NULL; + + ctx = kzalloc(sizeof(struct ecryptfs_fs_context), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + sbi = kmem_cache_zalloc(ecryptfs_sb_info_cache, GFP_KERNEL); + if (!sbi) { + kfree(ctx); + ctx = NULL; + return -ENOMEM; + } + + ecryptfs_init_mount_crypt_stat(&sbi->mount_crypt_stat); + + fc->fs_private = ctx; + fc->s_fs_info = sbi; + fc->ops = &ecryptfs_context_ops; + return 0; +} + static struct file_system_type ecryptfs_fs_type = { .owner = THIS_MODULE, .name = "ecryptfs", - .mount = ecryptfs_mount, + .init_fs_context = ecryptfs_init_fs_context, + .parameters = ecryptfs_fs_param_spec, .kill_sb = ecryptfs_kill_block_super, .fs_flags = 0 }; diff --git a/fs/exportfs/expfs.c b/fs/exportfs/expfs.c index 4f2dd4ab4486..0c899cfba578 100644 --- a/fs/exportfs/expfs.c +++ b/fs/exportfs/expfs.c @@ -382,14 +382,24 @@ int exportfs_encode_inode_fh(struct inode *inode, struct fid *fid, int *max_len, struct inode *parent, int flags) { const struct export_operations *nop = inode->i_sb->s_export_op; + enum fid_type type; if (!exportfs_can_encode_fh(nop, flags)) return -EOPNOTSUPP; if (!nop && (flags & EXPORT_FH_FID)) - return exportfs_encode_ino64_fid(inode, fid, max_len); + type = exportfs_encode_ino64_fid(inode, fid, max_len); + else + type = nop->encode_fh(inode, fid->raw, max_len, parent); + + if (type > 0 && FILEID_USER_FLAGS(type)) { + pr_warn_once("%s: unexpected fh type value 0x%x from fstype %s.\n", + __func__, type, inode->i_sb->s_type->name); + return -EINVAL; + } + + return type; - return nop->encode_fh(inode, fid->raw, max_len, parent); } EXPORT_SYMBOL_GPL(exportfs_encode_inode_fh); @@ -436,6 +446,9 @@ exportfs_decode_fh_raw(struct vfsmount *mnt, struct fid *fid, int fh_len, char nbuf[NAME_MAX+1]; int err; + if (fileid_type < 0 || FILEID_USER_FLAGS(fileid_type)) + return ERR_PTR(-EINVAL); + /* * Try to get any dentry for the given file handle from the filesystem. */ diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index 8bffdeccdbc3..1fbc0607363b 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -296,9 +296,8 @@ static struct posix_acl *f2fs_acl_clone(const struct posix_acl *acl, struct posix_acl *clone = NULL; if (acl) { - int size = sizeof(struct posix_acl) + acl->a_count * - sizeof(struct posix_acl_entry); - clone = kmemdup(acl, size, flags); + clone = kmemdup(acl, struct_size(acl, a_entries, acl->a_count), + flags); if (clone) refcount_set(&clone->a_refcount, 1); } diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 7f76460b721f..efda9a022981 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -32,7 +32,7 @@ void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, f2fs_build_fault_attr(sbi, 0, 0); if (!end_io) f2fs_flush_merged_writes(sbi); - f2fs_handle_critical_error(sbi, reason, end_io); + f2fs_handle_critical_error(sbi, reason); } /* diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index e3ce763cce18..a2478c2afb3a 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1679,7 +1679,8 @@ next_block: /* reserved delalloc block should be mapped for fiemap. */ if (blkaddr == NEW_ADDR) map->m_flags |= F2FS_MAP_DELALLOC; - if (flag != F2FS_GET_BLOCK_DIO || !is_hole) + /* DIO READ and hole case, should not map the blocks. */ + if (!(flag == F2FS_GET_BLOCK_DIO && is_hole && !map->m_may_create)) map->m_flags |= F2FS_MAP_MAPPED; map->m_pblk = blkaddr; @@ -1821,16 +1822,6 @@ bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) return true; } -static inline u64 bytes_to_blks(struct inode *inode, u64 bytes) -{ - return (bytes >> inode->i_blkbits); -} - -static inline u64 blks_to_bytes(struct inode *inode, u64 blks) -{ - return (blks << inode->i_blkbits); -} - static int f2fs_xattr_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo) { @@ -1856,7 +1847,7 @@ static int f2fs_xattr_fiemap(struct inode *inode, return err; } - phys = blks_to_bytes(inode, ni.blk_addr); + phys = F2FS_BLK_TO_BYTES(ni.blk_addr); offset = offsetof(struct f2fs_inode, i_addr) + sizeof(__le32) * (DEF_ADDRS_PER_INODE - get_inline_xattr_addrs(inode)); @@ -1888,7 +1879,7 @@ static int f2fs_xattr_fiemap(struct inode *inode, return err; } - phys = blks_to_bytes(inode, ni.blk_addr); + phys = F2FS_BLK_TO_BYTES(ni.blk_addr); len = inode->i_sb->s_blocksize; f2fs_put_page(page, 1); @@ -1904,30 +1895,11 @@ static int f2fs_xattr_fiemap(struct inode *inode, return (err < 0 ? err : 0); } -static loff_t max_inode_blocks(struct inode *inode) -{ - loff_t result = ADDRS_PER_INODE(inode); - loff_t leaf_count = ADDRS_PER_BLOCK(inode); - - /* two direct node blocks */ - result += (leaf_count * 2); - - /* two indirect node blocks */ - leaf_count *= NIDS_PER_BLOCK; - result += (leaf_count * 2); - - /* one double indirect node block */ - leaf_count *= NIDS_PER_BLOCK; - result += leaf_count; - - return result; -} - int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct f2fs_map_blocks map; - sector_t start_blk, last_blk; + sector_t start_blk, last_blk, blk_len, max_len; pgoff_t next_pgofs; u64 logical = 0, phys = 0, size = 0; u32 flags = 0; @@ -1969,16 +1941,15 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, goto out; } - if (bytes_to_blks(inode, len) == 0) - len = blks_to_bytes(inode, 1); - - start_blk = bytes_to_blks(inode, start); - last_blk = bytes_to_blks(inode, start + len - 1); + start_blk = F2FS_BYTES_TO_BLK(start); + last_blk = F2FS_BYTES_TO_BLK(start + len - 1); + blk_len = last_blk - start_blk + 1; + max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk; next: memset(&map, 0, sizeof(map)); map.m_lblk = start_blk; - map.m_len = bytes_to_blks(inode, len); + map.m_len = blk_len; map.m_next_pgofs = &next_pgofs; map.m_seg_type = NO_CHECK_TYPE; @@ -1995,13 +1966,23 @@ next: if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) { start_blk = next_pgofs; - if (blks_to_bytes(inode, start_blk) < blks_to_bytes(inode, - max_inode_blocks(inode))) + if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes) goto prep_next; flags |= FIEMAP_EXTENT_LAST; } + /* + * current extent may cross boundary of inquiry, increase len to + * requery. + */ + if (!compr_cluster && (map.m_flags & F2FS_MAP_MAPPED) && + map.m_lblk + map.m_len - 1 == last_blk && + blk_len != max_len) { + blk_len = max_len; + goto next; + } + compr_appended = false; /* In a case of compressed cluster, append this to the last extent */ if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) || @@ -2033,14 +2014,14 @@ skip_fill: } else if (compr_appended) { unsigned int appended_blks = cluster_size - count_in_cluster + 1; - size += blks_to_bytes(inode, appended_blks); + size += F2FS_BLK_TO_BYTES(appended_blks); start_blk += appended_blks; compr_cluster = false; } else { - logical = blks_to_bytes(inode, start_blk); + logical = F2FS_BLK_TO_BYTES(start_blk); phys = __is_valid_data_blkaddr(map.m_pblk) ? - blks_to_bytes(inode, map.m_pblk) : 0; - size = blks_to_bytes(inode, map.m_len); + F2FS_BLK_TO_BYTES(map.m_pblk) : 0; + size = F2FS_BLK_TO_BYTES(map.m_len); flags = 0; if (compr_cluster) { @@ -2048,13 +2029,13 @@ skip_fill: count_in_cluster += map.m_len; if (count_in_cluster == cluster_size) { compr_cluster = false; - size += blks_to_bytes(inode, 1); + size += F2FS_BLKSIZE; } } else if (map.m_flags & F2FS_MAP_DELALLOC) { flags = FIEMAP_EXTENT_UNWRITTEN; } - start_blk += bytes_to_blks(inode, size); + start_blk += F2FS_BYTES_TO_BLK(size); } prep_next: @@ -2092,7 +2073,7 @@ static int f2fs_read_single_page(struct inode *inode, struct folio *folio, struct readahead_control *rac) { struct bio *bio = *bio_ret; - const unsigned blocksize = blks_to_bytes(inode, 1); + const unsigned int blocksize = F2FS_BLKSIZE; sector_t block_in_file; sector_t last_block; sector_t last_block_in_file; @@ -2102,8 +2083,8 @@ static int f2fs_read_single_page(struct inode *inode, struct folio *folio, block_in_file = (sector_t)index; last_block = block_in_file + nr_pages; - last_block_in_file = bytes_to_blks(inode, - f2fs_readpage_limit(inode) + blocksize - 1); + last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + + blocksize - 1); if (last_block > last_block_in_file) last_block = last_block_in_file; @@ -2203,7 +2184,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, struct bio *bio = *bio_ret; unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; sector_t last_block_in_file; - const unsigned blocksize = blks_to_bytes(inode, 1); + const unsigned int blocksize = F2FS_BLKSIZE; struct decompress_io_ctx *dic = NULL; struct extent_info ei = {}; bool from_dnode = true; @@ -2212,8 +2193,8 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); - last_block_in_file = bytes_to_blks(inode, - f2fs_readpage_limit(inode) + blocksize - 1); + last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + + blocksize - 1); /* get rid of pages beyond EOF */ for (i = 0; i < cc->cluster_size; i++) { @@ -2388,10 +2369,10 @@ static int f2fs_mpage_readpages(struct inode *inode, .nr_cpages = 0, }; pgoff_t nc_cluster_idx = NULL_CLUSTER; + pgoff_t index; #endif unsigned nr_pages = rac ? readahead_count(rac) : 1; unsigned max_nr_pages = nr_pages; - pgoff_t index; int ret = 0; map.m_pblk = 0; @@ -2409,9 +2390,9 @@ static int f2fs_mpage_readpages(struct inode *inode, prefetchw(&folio->flags); } +#ifdef CONFIG_F2FS_FS_COMPRESSION index = folio_index(folio); -#ifdef CONFIG_F2FS_FS_COMPRESSION if (!f2fs_compressed_file(inode)) goto read_single_page; @@ -3444,6 +3425,11 @@ restart: if (!f2fs_lookup_read_extent_cache_block(inode, index, &dn.data_blkaddr)) { + if (IS_DEVICE_ALIASING(inode)) { + err = -ENODATA; + goto out; + } + if (locked) { err = f2fs_reserve_block(&dn, index); goto out; @@ -3974,7 +3960,7 @@ static int check_swap_activate(struct swap_info_struct *sis, * to be very smart. */ cur_lblock = 0; - last_lblock = bytes_to_blks(inode, i_size_read(inode)); + last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode)); while (cur_lblock < last_lblock && cur_lblock < sis->max) { struct f2fs_map_blocks map; @@ -4217,8 +4203,8 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, pgoff_t next_pgofs = 0; int err; - map.m_lblk = bytes_to_blks(inode, offset); - map.m_len = bytes_to_blks(inode, offset + length - 1) - map.m_lblk + 1; + map.m_lblk = F2FS_BYTES_TO_BLK(offset); + map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1; map.m_next_pgofs = &next_pgofs; map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), inode->i_write_hint); @@ -4229,7 +4215,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (err) return err; - iomap->offset = blks_to_bytes(inode, map.m_lblk); + iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk); /* * When inline encryption is enabled, sometimes I/O to an encrypted file @@ -4249,21 +4235,21 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) return -EINVAL; - iomap->length = blks_to_bytes(inode, map.m_len); + iomap->length = F2FS_BLK_TO_BYTES(map.m_len); iomap->type = IOMAP_MAPPED; iomap->flags |= IOMAP_F_MERGED; iomap->bdev = map.m_bdev; - iomap->addr = blks_to_bytes(inode, map.m_pblk); + iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk); } else { if (flags & IOMAP_WRITE) return -ENOTBLK; if (map.m_pblk == NULL_ADDR) { - iomap->length = blks_to_bytes(inode, next_pgofs) - - iomap->offset; + iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) - + iomap->offset; iomap->type = IOMAP_HOLE; } else if (map.m_pblk == NEW_ADDR) { - iomap->length = blks_to_bytes(inode, map.m_len); + iomap->length = F2FS_BLK_TO_BYTES(map.m_len); iomap->type = IOMAP_UNWRITTEN; } else { f2fs_bug_on(F2FS_I_SB(inode), 1); diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index 546b8ba91261..468828288a4a 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -60,6 +60,70 @@ void f2fs_update_sit_info(struct f2fs_sb_info *sbi) } #ifdef CONFIG_DEBUG_FS +static void update_multidevice_stats(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + struct f2fs_dev_stats *dev_stats = si->dev_stats; + int i, j; + + if (!f2fs_is_multi_device(sbi)) + return; + + memset(dev_stats, 0, sizeof(struct f2fs_dev_stats) * sbi->s_ndevs); + for (i = 0; i < sbi->s_ndevs; i++) { + unsigned int start_segno, end_segno; + block_t start_blk, end_blk; + + if (i == 0) { + start_blk = MAIN_BLKADDR(sbi); + end_blk = FDEV(i).end_blk + 1 - SEG0_BLKADDR(sbi); + } else { + start_blk = FDEV(i).start_blk; + end_blk = FDEV(i).end_blk + 1; + } + + start_segno = GET_SEGNO(sbi, start_blk); + end_segno = GET_SEGNO(sbi, end_blk); + + for (j = start_segno; j < end_segno; j++) { + unsigned int seg_blks, sec_blks; + + seg_blks = get_seg_entry(sbi, j)->valid_blocks; + + /* update segment stats */ + if (IS_CURSEG(sbi, j)) + dev_stats[i].devstats[0][DEVSTAT_INUSE]++; + else if (seg_blks == BLKS_PER_SEG(sbi)) + dev_stats[i].devstats[0][DEVSTAT_FULL]++; + else if (seg_blks != 0) + dev_stats[i].devstats[0][DEVSTAT_DIRTY]++; + else if (!test_bit(j, FREE_I(sbi)->free_segmap)) + dev_stats[i].devstats[0][DEVSTAT_FREE]++; + else + dev_stats[i].devstats[0][DEVSTAT_PREFREE]++; + + if (!__is_large_section(sbi) || + (j % SEGS_PER_SEC(sbi)) != 0) + continue; + + sec_blks = get_sec_entry(sbi, j)->valid_blocks; + + /* update section stats */ + if (IS_CURSEC(sbi, GET_SEC_FROM_SEG(sbi, j))) + dev_stats[i].devstats[1][DEVSTAT_INUSE]++; + else if (sec_blks == BLKS_PER_SEC(sbi)) + dev_stats[i].devstats[1][DEVSTAT_FULL]++; + else if (sec_blks != 0) + dev_stats[i].devstats[1][DEVSTAT_DIRTY]++; + else if (!test_bit(GET_SEC_FROM_SEG(sbi, j), + FREE_I(sbi)->free_secmap)) + dev_stats[i].devstats[1][DEVSTAT_FREE]++; + else + dev_stats[i].devstats[1][DEVSTAT_PREFREE]++; + } + } +} + static void update_general_status(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); @@ -214,6 +278,8 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->valid_blks[type] += blks; } + update_multidevice_stats(sbi); + for (i = 0; i < MAX_CALL_TYPE; i++) si->cp_call_count[i] = atomic_read(&sbi->cp_call_count[i]); @@ -498,6 +564,36 @@ static int stat_show(struct seq_file *s, void *v) si->dirty_count); seq_printf(s, " - Prefree: %d\n - Free: %d (%d)\n\n", si->prefree_count, si->free_segs, si->free_secs); + if (f2fs_is_multi_device(sbi)) { + seq_puts(s, "Multidevice stats:\n"); + seq_printf(s, " [seg: %8s %8s %8s %8s %8s]", + "inuse", "dirty", "full", "free", "prefree"); + if (__is_large_section(sbi)) + seq_printf(s, " [sec: %8s %8s %8s %8s %8s]\n", + "inuse", "dirty", "full", "free", "prefree"); + else + seq_puts(s, "\n"); + + for (i = 0; i < sbi->s_ndevs; i++) { + seq_printf(s, " #%-2d %8u %8u %8u %8u %8u", i, + si->dev_stats[i].devstats[0][DEVSTAT_INUSE], + si->dev_stats[i].devstats[0][DEVSTAT_DIRTY], + si->dev_stats[i].devstats[0][DEVSTAT_FULL], + si->dev_stats[i].devstats[0][DEVSTAT_FREE], + si->dev_stats[i].devstats[0][DEVSTAT_PREFREE]); + if (!__is_large_section(sbi)) { + seq_puts(s, "\n"); + continue; + } + seq_printf(s, " %8u %8u %8u %8u %8u\n", + si->dev_stats[i].devstats[1][DEVSTAT_INUSE], + si->dev_stats[i].devstats[1][DEVSTAT_DIRTY], + si->dev_stats[i].devstats[1][DEVSTAT_FULL], + si->dev_stats[i].devstats[1][DEVSTAT_FREE], + si->dev_stats[i].devstats[1][DEVSTAT_PREFREE]); + } + seq_puts(s, "\n"); + } seq_printf(s, "CP calls: %d (BG: %d)\n", si->cp_call_count[TOTAL_CALL], si->cp_call_count[BACKGROUND]); @@ -598,9 +694,9 @@ static int stat_show(struct seq_file *s, void *v) si->ndirty_node, si->node_pages); seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n", si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); - seq_printf(s, " - datas: %4d in files:%4d\n", + seq_printf(s, " - data: %4d in files:%4d\n", si->ndirty_data, si->ndirty_files); - seq_printf(s, " - quota datas: %4d in quota files:%4d\n", + seq_printf(s, " - quota data: %4d in quota files:%4d\n", si->ndirty_qdata, si->nquota_files); seq_printf(s, " - meta: %4d in %4d\n", si->ndirty_meta, si->meta_pages); @@ -665,6 +761,7 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); struct f2fs_stat_info *si; + struct f2fs_dev_stats *dev_stats; unsigned long flags; int i; @@ -672,6 +769,15 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi) if (!si) return -ENOMEM; + dev_stats = f2fs_kzalloc(sbi, sizeof(struct f2fs_dev_stats) * + sbi->s_ndevs, GFP_KERNEL); + if (!dev_stats) { + kfree(si); + return -ENOMEM; + } + + si->dev_stats = dev_stats; + si->all_area_segs = le32_to_cpu(raw_super->segment_count); si->sit_area_segs = le32_to_cpu(raw_super->segment_count_sit); si->nat_area_segs = le32_to_cpu(raw_super->segment_count_nat); @@ -724,6 +830,7 @@ void f2fs_destroy_stats(struct f2fs_sb_info *sbi) list_del(&si->stat_list); raw_spin_unlock_irqrestore(&f2fs_stat_lock, flags); + kfree(si->dev_stats); kfree(si); } diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 62ac440d9416..347b3b647834 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -24,6 +24,7 @@ bool sanity_check_extent_cache(struct inode *inode, struct page *ipage) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_extent *i_ext = &F2FS_INODE(ipage)->i_ext; struct extent_info ei; + int devi; get_read_extent_info(&ei, i_ext); @@ -38,7 +39,36 @@ bool sanity_check_extent_cache(struct inode *inode, struct page *ipage) ei.blk, ei.fofs, ei.len); return false; } - return true; + + if (!IS_DEVICE_ALIASING(inode)) + return true; + + for (devi = 0; devi < sbi->s_ndevs; devi++) { + if (FDEV(devi).start_blk != ei.blk || + FDEV(devi).end_blk != ei.blk + ei.len - 1) + continue; + + if (devi == 0) { + f2fs_warn(sbi, + "%s: inode (ino=%lx) is an alias of meta device", + __func__, inode->i_ino); + return false; + } + + if (bdev_is_zoned(FDEV(devi).bdev)) { + f2fs_warn(sbi, + "%s: device alias inode (ino=%lx)'s extent info " + "[%u, %u, %u] maps to zoned block device", + __func__, inode->i_ino, ei.blk, ei.fofs, ei.len); + return false; + } + return true; + } + + f2fs_warn(sbi, "%s: device alias inode (ino=%lx)'s extent info " + "[%u, %u, %u] is inconsistent w/ any devices", + __func__, inode->i_ino, ei.blk, ei.fofs, ei.len); + return false; } static void __set_extent_info(struct extent_info *ei, @@ -76,6 +106,9 @@ static bool __init_may_extent_tree(struct inode *inode, enum extent_type type) static bool __may_extent_tree(struct inode *inode, enum extent_type type) { + if (IS_DEVICE_ALIASING(inode) && type == EX_READ) + return true; + /* * for recovered files during mount do not create extents * if shrinker is not registered. @@ -346,21 +379,22 @@ static struct extent_tree *__grab_extent_tree(struct inode *inode, } static unsigned int __free_extent_tree(struct f2fs_sb_info *sbi, - struct extent_tree *et) + struct extent_tree *et, unsigned int nr_shrink) { struct rb_node *node, *next; struct extent_node *en; - unsigned int count = atomic_read(&et->node_cnt); + unsigned int count; node = rb_first_cached(&et->root); - while (node) { + + for (count = 0; node && count < nr_shrink; count++) { next = rb_next(node); en = rb_entry(node, struct extent_node, rb_node); __release_extent_node(sbi, et, en); node = next; } - return count - atomic_read(&et->node_cnt); + return count; } static void __drop_largest_extent(struct extent_tree *et, @@ -401,6 +435,11 @@ void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage) if (atomic_read(&et->node_cnt) || !ei.len) goto skip; + if (IS_DEVICE_ALIASING(inode)) { + et->largest = ei; + goto skip; + } + en = __attach_extent_node(sbi, et, &ei, NULL, &et->root.rb_root.rb_node, true); if (en) { @@ -463,6 +502,11 @@ static bool __lookup_extent_tree(struct inode *inode, pgoff_t pgofs, goto out; } + if (IS_DEVICE_ALIASING(inode)) { + ret = false; + goto out; + } + en = __lookup_extent_node(&et->root, et->cached_en, pgofs); if (!en) goto out; @@ -579,6 +623,30 @@ do_insert: return en; } +static unsigned int __destroy_extent_node(struct inode *inode, + enum extent_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; + unsigned int nr_shrink = type == EX_READ ? + READ_EXTENT_CACHE_SHRINK_NUMBER : + AGE_EXTENT_CACHE_SHRINK_NUMBER; + unsigned int node_cnt = 0; + + if (!et || !atomic_read(&et->node_cnt)) + return 0; + + while (atomic_read(&et->node_cnt)) { + write_lock(&et->lock); + node_cnt += __free_extent_tree(sbi, et, nr_shrink); + write_unlock(&et->lock); + } + + f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); + + return node_cnt; +} + static void __update_extent_tree_range(struct inode *inode, struct extent_info *tei, enum extent_type type) { @@ -649,7 +717,9 @@ static void __update_extent_tree_range(struct inode *inode, } if (end < org_end && (type != EX_READ || - org_end - end >= F2FS_MIN_EXTENT_LEN)) { + (org_end - end >= F2FS_MIN_EXTENT_LEN && + atomic_read(&et->node_cnt) < + sbi->max_read_extent_count))) { if (parts) { __set_extent_info(&ei, end, org_end - end, @@ -717,9 +787,6 @@ static void __update_extent_tree_range(struct inode *inode, } } - if (is_inode_flag_set(inode, FI_NO_EXTENT)) - __free_extent_tree(sbi, et); - if (et->largest_updated) { et->largest_updated = false; updated = true; @@ -737,6 +804,9 @@ update_age_extent_cache: out_read_extent_cache: write_unlock(&et->lock); + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + __destroy_extent_node(inode, EX_READ); + if (updated) f2fs_mark_inode_dirty_sync(inode, true); } @@ -899,10 +969,14 @@ static unsigned int __shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink list_for_each_entry_safe(et, next, &eti->zombie_list, list) { if (atomic_read(&et->node_cnt)) { write_lock(&et->lock); - node_cnt += __free_extent_tree(sbi, et); + node_cnt += __free_extent_tree(sbi, et, + nr_shrink - node_cnt - tree_cnt); write_unlock(&et->lock); } - f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); + + if (atomic_read(&et->node_cnt)) + goto unlock_out; + list_del_init(&et->list); radix_tree_delete(&eti->extent_tree_root, et->ino); kmem_cache_free(extent_tree_slab, et); @@ -1041,23 +1115,6 @@ unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink return __shrink_extent_tree(sbi, nr_shrink, EX_BLOCK_AGE); } -static unsigned int __destroy_extent_node(struct inode *inode, - enum extent_type type) -{ - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; - unsigned int node_cnt = 0; - - if (!et || !atomic_read(&et->node_cnt)) - return 0; - - write_lock(&et->lock); - node_cnt = __free_extent_tree(sbi, et); - write_unlock(&et->lock); - - return node_cnt; -} - void f2fs_destroy_extent_node(struct inode *inode) { __destroy_extent_node(inode, EX_READ); @@ -1066,7 +1123,6 @@ void f2fs_destroy_extent_node(struct inode *inode) static void __drop_extent_tree(struct inode *inode, enum extent_type type) { - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; bool updated = false; @@ -1074,7 +1130,6 @@ static void __drop_extent_tree(struct inode *inode, enum extent_type type) return; write_lock(&et->lock); - __free_extent_tree(sbi, et); if (type == EX_READ) { set_inode_flag(inode, FI_NO_EXTENT); if (et->largest.len) { @@ -1083,6 +1138,9 @@ static void __drop_extent_tree(struct inode *inode, enum extent_type type) } } write_unlock(&et->lock); + + __destroy_extent_node(inode, type); + if (updated) f2fs_mark_inode_dirty_sync(inode, true); } @@ -1156,6 +1214,7 @@ void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi) sbi->hot_data_age_threshold = DEF_HOT_DATA_AGE_THRESHOLD; sbi->warm_data_age_threshold = DEF_WARM_DATA_AGE_THRESHOLD; sbi->last_age_weight = LAST_AGE_WEIGHT; + sbi->max_read_extent_count = DEF_MAX_READ_EXTENT_COUNT; } int __init f2fs_create_extent_cache(void) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 33f5449dc22d..6f2cbf4c5740 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -213,6 +213,7 @@ struct f2fs_mount_info { #define F2FS_FEATURE_CASEFOLD 0x00001000 #define F2FS_FEATURE_COMPRESSION 0x00002000 #define F2FS_FEATURE_RO 0x00004000 +#define F2FS_FEATURE_DEVICE_ALIAS 0x00008000 #define __F2FS_HAS_FEATURE(raw_super, mask) \ ((raw_super->feature & cpu_to_le32(mask)) != 0) @@ -634,6 +635,9 @@ enum { #define DEF_HOT_DATA_AGE_THRESHOLD 262144 #define DEF_WARM_DATA_AGE_THRESHOLD 2621440 +/* default max read extent count per inode */ +#define DEF_MAX_READ_EXTENT_COUNT 10240 + /* extent cache type */ enum extent_type { EX_READ, @@ -1018,7 +1022,7 @@ static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode, #define NR_CURSEG_PERSIST_TYPE (NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE) #define NR_CURSEG_TYPE (NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE) -enum { +enum log_type { CURSEG_HOT_DATA = 0, /* directory entry blocks */ CURSEG_WARM_DATA, /* data blocks */ CURSEG_COLD_DATA, /* multimedia or GCed data blocks */ @@ -1063,7 +1067,6 @@ struct f2fs_sm_info { unsigned int segment_count; /* total # of segments */ unsigned int main_segments; /* # of segments in main area */ unsigned int reserved_segments; /* # of reserved segments */ - unsigned int additional_reserved_segments;/* reserved segs for IO align feature */ unsigned int ovp_segments; /* # of overprovision segments */ /* a threshold to reclaim prefree segments */ @@ -1619,6 +1622,7 @@ struct f2fs_sb_info { /* for extent tree cache */ struct extent_tree_info extent_tree[NR_EXTENT_CACHES]; atomic64_t allocated_data_blocks; /* for block age extent_cache */ + unsigned int max_read_extent_count; /* max read extent count per inode */ /* The threshold used for hot and warm data seperation*/ unsigned int hot_data_age_threshold; @@ -1758,6 +1762,7 @@ struct f2fs_sb_info { unsigned int dirty_device; /* for checkpoint data flush */ spinlock_t dev_lock; /* protect dirty_device */ bool aligned_blksize; /* all devices has the same logical blksize */ + unsigned int first_zoned_segno; /* first zoned segno */ /* For write statistics */ u64 sectors_written_start; @@ -3046,6 +3051,7 @@ static inline void f2fs_change_bit(unsigned int nr, char *addr) #define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */ #define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */ #define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */ +#define F2FS_DEVICE_ALIAS_FL 0x80000000 /* File for aliasing a device */ #define F2FS_QUOTA_DEFAULT_FL (F2FS_NOATIME_FL | F2FS_IMMUTABLE_FL) @@ -3061,6 +3067,8 @@ static inline void f2fs_change_bit(unsigned int nr, char *addr) /* Flags that are appropriate for non-directories/regular files. */ #define F2FS_OTHER_FLMASK (F2FS_NODUMP_FL | F2FS_NOATIME_FL) +#define IS_DEVICE_ALIASING(inode) (F2FS_I(inode)->i_flags & F2FS_DEVICE_ALIAS_FL) + static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags) { if (S_ISDIR(mode)) @@ -3632,8 +3640,7 @@ int f2fs_quota_sync(struct super_block *sb, int type); loff_t max_file_blocks(struct inode *inode); void f2fs_quota_off_umount(struct super_block *sb); void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag); -void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason, - bool irq_context); +void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason); void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error); void f2fs_handle_error_async(struct f2fs_sb_info *sbi, unsigned char error); int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover); @@ -3754,7 +3761,8 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, block_t old_addr, block_t new_addr, unsigned char version, bool recover_curseg, bool recover_newaddr); -int f2fs_get_segment_temp(int seg_type); +enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, + enum log_type seg_type); int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, block_t old_blkaddr, block_t *new_blkaddr, struct f2fs_summary *sum, int type, @@ -3771,8 +3779,7 @@ void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk); int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, unsigned int val, int alloc); void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); -int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi); -int f2fs_check_write_pointer(struct f2fs_sb_info *sbi); +int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi); int f2fs_build_segment_manager(struct f2fs_sb_info *sbi); void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi); int __init f2fs_create_segment_manager_caches(void); @@ -3783,6 +3790,8 @@ enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi); unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, unsigned int segno); +unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi, + unsigned int segno); #define DEF_FRAGMENT_SIZE 4 #define MIN_FRAGMENT_SIZE 1 @@ -3935,6 +3944,19 @@ void f2fs_destroy_recovery_cache(void); * debug.c */ #ifdef CONFIG_F2FS_STAT_FS +enum { + DEVSTAT_INUSE, + DEVSTAT_DIRTY, + DEVSTAT_FULL, + DEVSTAT_FREE, + DEVSTAT_PREFREE, + DEVSTAT_MAX, +}; + +struct f2fs_dev_stats { + unsigned int devstats[2][DEVSTAT_MAX]; /* 0: segs, 1: secs */ +}; + struct f2fs_stat_info { struct list_head stat_list; struct f2fs_sb_info *sbi; @@ -3998,6 +4020,7 @@ struct f2fs_stat_info { unsigned int block_count[2]; unsigned int inplace_count; unsigned long long base_mem, cache_mem, page_mem; + struct f2fs_dev_stats *dev_stats; }; static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi) @@ -4510,6 +4533,7 @@ F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM); F2FS_FEATURE_FUNCS(casefold, CASEFOLD); F2FS_FEATURE_FUNCS(compression, COMPRESSION); F2FS_FEATURE_FUNCS(readonly, RO); +F2FS_FEATURE_FUNCS(device_alias, DEVICE_ALIAS); #ifdef CONFIG_BLK_DEV_ZONED static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi, diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 84447d5145aa..aa9679b3d8e4 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -725,6 +725,11 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) trace_f2fs_truncate_blocks_enter(inode, from); + if (IS_DEVICE_ALIASING(inode) && from) { + err = -EINVAL; + goto out_err; + } + free_from = (pgoff_t)F2FS_BLK_ALIGN(from); if (free_from >= max_file_blocks(inode)) @@ -739,6 +744,21 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) goto out; } + if (IS_DEVICE_ALIASING(inode)) { + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; + struct extent_info ei = et->largest; + unsigned int i; + + for (i = 0; i < ei.len; i++) + f2fs_invalidate_blocks(sbi, ei.blk + i); + + dec_valid_block_count(sbi, inode, ei.len); + f2fs_update_time(sbi, REQ_TIME); + + f2fs_put_page(ipage, 1); + goto out; + } + if (f2fs_has_inline_data(inode)) { f2fs_truncate_inline_inode(inode, ipage, from); f2fs_put_page(ipage, 1); @@ -774,7 +794,7 @@ free_partial: /* lastly zero out the first data page */ if (!err) err = truncate_partial_data_page(inode, from, truncate_page); - +out_err: trace_f2fs_truncate_blocks_exit(inode, err); return err; } @@ -863,7 +883,11 @@ static bool f2fs_force_buffered_io(struct inode *inode, int rw) return true; if (f2fs_compressed_file(inode)) return true; - if (f2fs_has_inline_data(inode)) + /* + * only force direct read to use buffered IO, for direct write, + * it expects inline data conversion before committing IO. + */ + if (f2fs_has_inline_data(inode) && rw == READ) return true; /* disallow direct IO if any of devices has unaligned blksize */ @@ -992,7 +1016,8 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; if ((attr->ia_valid & ATTR_SIZE)) { - if (!f2fs_is_compress_backend_ready(inode)) + if (!f2fs_is_compress_backend_ready(inode) || + IS_DEVICE_ALIASING(inode)) return -EOPNOTSUPP; if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) && !IS_ALIGNED(attr->ia_size, @@ -1790,7 +1815,8 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, map.m_len = sec_blks; next_alloc: - if (has_not_enough_free_secs(sbi, 0, + if (has_not_enough_free_secs(sbi, 0, f2fs_sb_has_blkzoned(sbi) ? + ZONED_PIN_SEC_REQUIRED_COUNT : GET_SEC_FROM_SEG(sbi, overprovision_segments(sbi)))) { f2fs_down_write(&sbi->gc_lock); stat_inc_gc_call_count(sbi, FOREGROUND); @@ -1860,7 +1886,7 @@ static long f2fs_fallocate(struct file *file, int mode, return -EIO; if (!f2fs_is_checkpoint_ready(F2FS_I_SB(inode))) return -ENOSPC; - if (!f2fs_is_compress_backend_ready(inode)) + if (!f2fs_is_compress_backend_ready(inode) || IS_DEVICE_ALIASING(inode)) return -EOPNOTSUPP; /* f2fs only support ->fallocate for regular file */ @@ -2343,9 +2369,12 @@ int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, if (readonly) goto out; - /* grab sb->s_umount to avoid racing w/ remount() */ + /* + * grab sb->s_umount to avoid racing w/ remount() and other shutdown + * paths. + */ if (need_lock) - down_read(&sbi->sb->s_umount); + down_write(&sbi->sb->s_umount); f2fs_stop_gc_thread(sbi); f2fs_stop_discard_thread(sbi); @@ -2354,7 +2383,7 @@ int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, clear_opt(sbi, DISCARD); if (need_lock) - up_read(&sbi->sb->s_umount); + up_write(&sbi->sb->s_umount); f2fs_update_time(sbi, REQ_TIME); out: @@ -2861,7 +2890,7 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (!S_ISREG(inode->i_mode) || f2fs_is_atomic_file(inode)) + if (!S_ISREG(inode->i_mode)) return -EINVAL; if (f2fs_readonly(sbi->sb)) @@ -3291,6 +3320,9 @@ int f2fs_pin_file_control(struct inode *inode, bool inc) struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + if (IS_DEVICE_ALIASING(inode)) + return -EINVAL; + if (fi->i_gc_failures >= sbi->gc_pin_file_threshold) { f2fs_warn(sbi, "%s: Enable GC = ino %lx after %x GC trials", __func__, inode->i_ino, fi->i_gc_failures); @@ -3321,6 +3353,9 @@ static int f2fs_ioc_set_pin_file(struct file *filp, unsigned long arg) if (f2fs_readonly(sbi->sb)) return -EROFS; + if (!pin && IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + ret = mnt_want_write_file(filp); if (ret) return ret; @@ -3386,6 +3421,12 @@ static int f2fs_ioc_get_pin_file(struct file *filp, unsigned long arg) return put_user(pin, (u32 __user *)arg); } +static int f2fs_ioc_get_dev_alias_file(struct file *filp, unsigned long arg) +{ + return put_user(IS_DEVICE_ALIASING(file_inode(filp)) ? 1 : 0, + (u32 __user *)arg); +} + int f2fs_precache_extents(struct inode *inode) { struct f2fs_inode_info *fi = F2FS_I(inode); @@ -3787,7 +3828,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, to_reserved = cluster_size - compr_blocks - reserved; /* for the case all blocks in cluster were reserved */ - if (to_reserved == 1) { + if (reserved && to_reserved == 1) { dn->ofs_in_node += cluster_size; goto next; } @@ -4485,6 +4526,8 @@ static long __f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return f2fs_ioc_decompress_file(filp); case F2FS_IOC_COMPRESS_FILE: return f2fs_ioc_compress_file(filp); + case F2FS_IOC_GET_DEV_ALIAS_FILE: + return f2fs_ioc_get_dev_alias_file(filp, arg); default: return -ENOTTY; } @@ -4760,7 +4803,8 @@ static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter, else return 0; - map.m_may_create = true; + if (!IS_DEVICE_ALIASING(inode)) + map.m_may_create = true; if (dio) { map.m_seg_type = f2fs_rw_hint_to_seg_type(sbi, inode->i_write_hint); @@ -4816,8 +4860,8 @@ static void f2fs_dio_write_submit_io(const struct iomap_iter *iter, { struct inode *inode = iter->inode; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - int seg_type = f2fs_rw_hint_to_seg_type(sbi, inode->i_write_hint); - enum temp_type temp = f2fs_get_segment_temp(seg_type); + enum log_type type = f2fs_rw_hint_to_seg_type(sbi, inode->i_write_hint); + enum temp_type temp = f2fs_get_segment_temp(sbi, type); bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, DATA, temp); submit_bio(bio); @@ -5197,6 +5241,7 @@ long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case F2FS_IOC_SET_COMPRESS_OPTION: case F2FS_IOC_DECOMPRESS_FILE: case F2FS_IOC_COMPRESS_FILE: + case F2FS_IOC_GET_DEV_ALIAS_FILE: break; default: return -ENOIOCTLCMD; diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 9322a7200e31..3e1b6d2ff3a7 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -257,6 +257,8 @@ static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type) switch (sbi->gc_mode) { case GC_IDLE_CB: + case GC_URGENT_LOW: + case GC_URGENT_MID: gc_mode = GC_CB; break; case GC_IDLE_GREEDY: @@ -361,20 +363,15 @@ static unsigned int check_bg_victims(struct f2fs_sb_info *sbi) static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno) { struct sit_info *sit_i = SIT_I(sbi); - unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); - unsigned int start = GET_SEG_FROM_SEC(sbi, secno); unsigned long long mtime = 0; unsigned int vblocks; unsigned char age = 0; unsigned char u; - unsigned int i; unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi); - for (i = 0; i < usable_segs_per_sec; i++) - mtime += get_seg_entry(sbi, start + i)->mtime; + mtime = f2fs_get_section_mtime(sbi, segno); + f2fs_bug_on(sbi, mtime == INVALID_MTIME); vblocks = get_valid_blocks(sbi, segno, true); - - mtime = div_u64(mtime, usable_segs_per_sec); vblocks = div_u64(vblocks, usable_segs_per_sec); u = BLKS_TO_SEGS(sbi, vblocks * 100); @@ -519,10 +516,7 @@ static void add_victim_entry(struct f2fs_sb_info *sbi, struct victim_sel_policy *p, unsigned int segno) { struct sit_info *sit_i = SIT_I(sbi); - unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); - unsigned int start = GET_SEG_FROM_SEC(sbi, secno); unsigned long long mtime = 0; - unsigned int i; if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { if (p->gc_mode == GC_AT && @@ -530,9 +524,8 @@ static void add_victim_entry(struct f2fs_sb_info *sbi, return; } - for (i = 0; i < SEGS_PER_SEC(sbi); i++) - mtime += get_seg_entry(sbi, start + i)->mtime; - mtime = div_u64(mtime, SEGS_PER_SEC(sbi)); + mtime = f2fs_get_section_mtime(sbi, segno); + f2fs_bug_on(sbi, mtime == INVALID_MTIME); /* Handle if the system time has changed by the user */ if (mtime < sit_i->min_mtime) diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h index 2914b678bf8f..5c1eaf55e127 100644 --- a/fs/f2fs/gc.h +++ b/fs/f2fs/gc.h @@ -35,6 +35,7 @@ #define LIMIT_BOOST_ZONED_GC 25 /* percentage over total user space of boosted gc for zoned devices */ #define DEF_MIGRATION_WINDOW_GRANULARITY_ZONED 3 #define BOOST_GC_MULTIPLE 5 +#define ZONED_PIN_SEC_REQUIRED_COUNT 1 #define DEF_GC_FAILED_PINNED_FILES 2048 #define MAX_GC_FAILED_PINNED_FILES USHRT_MAX diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 1ed86df343a5..282fd320bdb3 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -372,6 +372,19 @@ static bool sanity_check_inode(struct inode *inode, struct page *node_page) return false; } + if (IS_DEVICE_ALIASING(inode)) { + if (!f2fs_sb_has_device_alias(sbi)) { + f2fs_warn(sbi, "%s: inode (ino=%lx) has device alias flag, but the feature is off", + __func__, inode->i_ino); + return false; + } + if (!f2fs_is_pinned_file(inode)) { + f2fs_warn(sbi, "%s: inode (ino=%lx) has device alias flag, but is not pinned", + __func__, inode->i_ino); + return false; + } + } + return true; } @@ -775,8 +788,10 @@ int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) !is_inode_flag_set(inode, FI_DIRTY_INODE)) return 0; - if (!f2fs_is_checkpoint_ready(sbi)) + if (!f2fs_is_checkpoint_ready(sbi)) { + f2fs_mark_inode_dirty_sync(inode, true); return -ENOSPC; + } /* * We need to balance fs here to prevent from producing dirty node pages @@ -823,7 +838,8 @@ void f2fs_evict_inode(struct inode *inode) f2fs_bug_on(sbi, get_dirty_pages(inode)); f2fs_remove_dirty_inode(inode); - f2fs_destroy_extent_tree(inode); + if (!IS_DEVICE_ALIASING(inode)) + f2fs_destroy_extent_tree(inode); if (inode->i_nlink || is_bad_inode(inode)) goto no_delete; @@ -879,6 +895,9 @@ retry: goto retry; } + if (IS_DEVICE_ALIASING(inode)) + f2fs_destroy_extent_tree(inode); + if (err) { f2fs_update_inode_page(inode); if (dquot_initialize_needed(inode)) diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 59b13ff243fa..0b900a7a48e5 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -905,6 +905,16 @@ static int truncate_node(struct dnode_of_data *dn) if (err) return err; + if (ni.blk_addr != NEW_ADDR && + !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) { + f2fs_err_ratelimited(sbi, + "nat entry is corrupted, run fsck to fix it, ino:%u, " + "nid:%u, blkaddr:%u", ni.ino, ni.nid, ni.blk_addr); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + return -EFSCORRUPTED; + } + /* Deallocate node address */ f2fs_invalidate_blocks(sbi, ni.blk_addr); dec_valid_node_count(sbi, dn->inode, dn->nid == dn->inode->i_ino); @@ -1056,7 +1066,7 @@ static int truncate_partial_nodes(struct dnode_of_data *dn, int i; int idx = depth - 2; - nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]); + nid[0] = get_nid(dn->inode_page, offset[0], true); if (!nid[0]) return 0; @@ -1167,7 +1177,7 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) skip_partial: while (cont) { - dn.nid = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]); + dn.nid = get_nid(page, offset[0], true); switch (offset[0]) { case NODE_DIR1_BLOCK: case NODE_DIR2_BLOCK: @@ -1199,13 +1209,10 @@ skip_partial: } if (err < 0) goto fail; - if (offset[1] == 0 && - ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) { + if (offset[1] == 0 && get_nid(page, offset[0], true)) { lock_page(page); BUG_ON(page->mapping != NODE_MAPPING(sbi)); - f2fs_wait_on_page_writeback(page, NODE, true, true); - ri->i_nid[offset[0] - NODE_DIR1_BLOCK] = 0; - set_page_dirty(page); + set_nid(page, offset[0], 0, true); unlock_page(page); } offset[1] = 0; @@ -1331,7 +1338,12 @@ struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs) err = -EFSCORRUPTED; dec_valid_node_count(sbi, dn->inode, !ofs); set_sbi_flag(sbi, SBI_NEED_FSCK); - f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); + f2fs_warn_ratelimited(sbi, + "f2fs_new_node_page: inconsistent nat entry, " + "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", + new_ni.ino, new_ni.nid, new_ni.blk_addr, + new_ni.version, new_ni.flag); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); goto fail; } #endif diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index e4d81b8705d1..f35be2c48e3c 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -899,13 +899,8 @@ skip: * and the f2fs is not read only, check and fix zoned block devices' * write pointer consistency. */ - if (f2fs_sb_has_blkzoned(sbi) && !f2fs_readonly(sbi->sb)) { - int err2 = f2fs_fix_curseg_write_pointer(sbi); - - if (!err2) - err2 = f2fs_check_write_pointer(sbi); - if (err2) - err = err2; + if (!err) { + err = f2fs_check_and_fix_write_pointer(sbi); ret = err; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 1766254279d2..eade36c5ef13 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -1290,16 +1290,18 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi, wait_list, issued); return 0; } - - /* - * Issue discard for conventional zones only if the device - * supports discard. - */ - if (!bdev_max_discard_sectors(bdev)) - return -EOPNOTSUPP; } #endif + /* + * stop issuing discard for any of below cases: + * 1. device is conventional zone, but it doesn't support discard. + * 2. device is regulare device, after snapshot it doesn't support + * discard. + */ + if (!bdev_max_discard_sectors(bdev)) + return -EOPNOTSUPP; + trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len); lstart = dc->di.lstart; @@ -2711,7 +2713,7 @@ static int get_new_segment(struct f2fs_sb_info *sbi, if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV || pinning) segno = 0; else - segno = max(first_zoned_segno(sbi), *newseg); + segno = max(sbi->first_zoned_segno, *newseg); hint = GET_SEC_FROM_SEG(sbi, segno); } #endif @@ -2723,7 +2725,7 @@ find_other_zone: if (secno >= MAIN_SECS(sbi) && f2fs_sb_has_blkzoned(sbi)) { /* Write only to sequential zones */ if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) { - hint = GET_SEC_FROM_SEG(sbi, first_zoned_segno(sbi)); + hint = GET_SEC_FROM_SEG(sbi, sbi->first_zoned_segno); secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); } else secno = find_first_zero_bit(free_i->free_secmap, @@ -2926,7 +2928,8 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) struct f2fs_summary_block *sum_node; struct page *sum_page; - write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno)); + if (curseg->inited) + write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno)); __set_test_and_inuse(sbi, new_segno); @@ -3237,7 +3240,8 @@ retry: if (f2fs_sb_has_blkzoned(sbi) && err == -EAGAIN && gc_required) { f2fs_down_write(&sbi->gc_lock); - err = f2fs_gc_range(sbi, 0, GET_SEGNO(sbi, FDEV(0).end_blk), true, 1); + err = f2fs_gc_range(sbi, 0, GET_SEGNO(sbi, FDEV(0).end_blk), + true, ZONED_PIN_SEC_REQUIRED_COUNT); f2fs_up_write(&sbi->gc_lock); gc_required = false; @@ -3581,18 +3585,35 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) } } -int f2fs_get_segment_temp(int seg_type) +enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, + enum log_type type) { - if (IS_HOT(seg_type)) - return HOT; - else if (IS_WARM(seg_type)) - return WARM; - return COLD; + struct curseg_info *curseg = CURSEG_I(sbi, type); + enum temp_type temp = COLD; + + switch (curseg->seg_type) { + case CURSEG_HOT_NODE: + case CURSEG_HOT_DATA: + temp = HOT; + break; + case CURSEG_WARM_NODE: + case CURSEG_WARM_DATA: + temp = WARM; + break; + case CURSEG_COLD_NODE: + case CURSEG_COLD_DATA: + temp = COLD; + break; + default: + f2fs_bug_on(sbi, 1); + } + + return temp; } static int __get_segment_type(struct f2fs_io_info *fio) { - int type = 0; + enum log_type type = CURSEG_HOT_DATA; switch (F2FS_OPTION(fio->sbi).active_logs) { case 2: @@ -3608,7 +3629,7 @@ static int __get_segment_type(struct f2fs_io_info *fio) f2fs_bug_on(fio->sbi, true); } - fio->temp = f2fs_get_segment_temp(type); + fio->temp = f2fs_get_segment_temp(fio->sbi, type); return type; } @@ -3793,10 +3814,35 @@ void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, } } +static int log_type_to_seg_type(enum log_type type) +{ + int seg_type = CURSEG_COLD_DATA; + + switch (type) { + case CURSEG_HOT_DATA: + case CURSEG_WARM_DATA: + case CURSEG_COLD_DATA: + case CURSEG_HOT_NODE: + case CURSEG_WARM_NODE: + case CURSEG_COLD_NODE: + seg_type = (int)type; + break; + case CURSEG_COLD_DATA_PINNED: + case CURSEG_ALL_DATA_ATGC: + seg_type = CURSEG_COLD_DATA; + break; + default: + break; + } + return seg_type; +} + static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) { - int type = __get_segment_type(fio); - bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA); + enum log_type type = __get_segment_type(fio); + int seg_type = log_type_to_seg_type(type); + bool keep_order = (f2fs_lfs_mode(fio->sbi) && + seg_type == CURSEG_COLD_DATA); if (keep_order) f2fs_down_read(&fio->sbi->io_order_lock); @@ -3977,8 +4023,8 @@ void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } } - f2fs_bug_on(sbi, !IS_DATASEG(type)); curseg = CURSEG_I(sbi, type); + f2fs_bug_on(sbi, !IS_DATASEG(curseg->seg_type)); mutex_lock(&curseg->curseg_mutex); down_write(&sit_i->sentry_lock); @@ -4778,12 +4824,7 @@ static int build_curseg(struct f2fs_sb_info *sbi) sizeof(struct f2fs_journal), GFP_KERNEL); if (!array[i].journal) return -ENOMEM; - if (i < NR_PERSISTENT_LOG) - array[i].seg_type = CURSEG_HOT_DATA + i; - else if (i == CURSEG_COLD_DATA_PINNED) - array[i].seg_type = CURSEG_COLD_DATA; - else if (i == CURSEG_ALL_DATA_ATGC) - array[i].seg_type = CURSEG_COLD_DATA; + array[i].seg_type = log_type_to_seg_type(i); reset_curseg_fields(&array[i]); } return restore_curseg_summaries(sbi); @@ -5207,7 +5248,7 @@ static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx, return 0; } -static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) +static int do_fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) { struct curseg_info *cs = CURSEG_I(sbi, type); struct f2fs_dev_info *zbd; @@ -5312,12 +5353,12 @@ static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) return 0; } -int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) +static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi) { int i, ret; for (i = 0; i < NR_PERSISTENT_LOG; i++) { - ret = fix_curseg_write_pointer(sbi, i); + ret = do_fix_curseg_write_pointer(sbi, i); if (ret) return ret; } @@ -5340,7 +5381,7 @@ static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx, return check_zone_write_pointer(args->sbi, args->fdev, zone); } -int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) +static int check_write_pointer(struct f2fs_sb_info *sbi) { int i, ret; struct check_zone_write_pointer_args args; @@ -5360,6 +5401,20 @@ int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) return 0; } +int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) +{ + int ret; + + if (!f2fs_sb_has_blkzoned(sbi) || f2fs_readonly(sbi->sb)) + return 0; + + f2fs_notice(sbi, "Checking entire write pointers"); + ret = fix_curseg_write_pointer(sbi); + if (!ret) + ret = check_write_pointer(sbi); + return ret; +} + /* * Return the number of usable blocks in a segment. The number of blocks * returned is always equal to the number of blocks in a segment for @@ -5396,12 +5451,7 @@ static inline unsigned int f2fs_usable_zone_blks_in_seg( return BLKS_PER_SEG(sbi); } #else -int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) -{ - return 0; -} - -int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) +int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) { return 0; } @@ -5430,6 +5480,35 @@ unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi) return SEGS_PER_SEC(sbi); } +unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi); + unsigned int secno = 0, start = 0; + unsigned int total_valid_blocks = 0; + unsigned long long mtime = 0; + unsigned int i = 0; + + secno = GET_SEC_FROM_SEG(sbi, segno); + start = GET_SEG_FROM_SEC(sbi, secno); + + if (!__is_large_section(sbi)) + return get_seg_entry(sbi, start + i)->mtime; + + for (i = 0; i < usable_segs_per_sec; i++) { + /* for large section, only check the mtime of valid segments */ + struct seg_entry *se = get_seg_entry(sbi, start+i); + + mtime += se->mtime * se->valid_blocks; + total_valid_blocks += se->valid_blocks; + } + + if (total_valid_blocks == 0) + return INVALID_MTIME; + + return div_u64(mtime, total_valid_blocks); +} + /* * Update min, max modified time for cost-benefit GC algorithm */ @@ -5443,13 +5522,9 @@ static void init_min_max_mtime(struct f2fs_sb_info *sbi) sit_i->min_mtime = ULLONG_MAX; for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { - unsigned int i; unsigned long long mtime = 0; - for (i = 0; i < SEGS_PER_SEC(sbi); i++) - mtime += get_seg_entry(sbi, segno + i)->mtime; - - mtime = div_u64(mtime, SEGS_PER_SEC(sbi)); + mtime = f2fs_get_section_mtime(sbi, segno); if (sit_i->min_mtime > mtime) sit_i->min_mtime = mtime; diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 71adb4a43bec..943be4f1d6d2 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -18,6 +18,8 @@ #define F2FS_MIN_SEGMENTS 9 /* SB + 2 (CP + SIT + NAT) + SSA + MAIN */ #define F2FS_MIN_META_SEGMENTS 8 /* SB + 2 (CP + SIT + NAT) + SSA */ +#define INVALID_MTIME ULLONG_MAX /* no valid blocks in a segment/section */ + /* L: Logical segment # in volume, R: Relative segment # in main area */ #define GET_L2R_SEGNO(free_i, segno) ((segno) - (free_i)->start_segno) #define GET_R2L_SEGNO(free_i, segno) ((segno) + (free_i)->start_segno) @@ -32,10 +34,6 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, seg_type >= NR_PERSISTENT_LOG); } -#define IS_HOT(t) ((t) == CURSEG_HOT_NODE || (t) == CURSEG_HOT_DATA) -#define IS_WARM(t) ((t) == CURSEG_WARM_NODE || (t) == CURSEG_WARM_DATA) -#define IS_COLD(t) ((t) == CURSEG_COLD_NODE || (t) == CURSEG_COLD_DATA) - #define IS_CURSEG(sbi, seg) \ (((seg) == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno) || \ ((seg) == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno) || \ @@ -524,8 +522,7 @@ static inline unsigned int free_segments(struct f2fs_sb_info *sbi) static inline unsigned int reserved_segments(struct f2fs_sb_info *sbi) { - return SM_I(sbi)->reserved_segments + - SM_I(sbi)->additional_reserved_segments; + return SM_I(sbi)->reserved_segments; } static inline unsigned int free_sections(struct f2fs_sb_info *sbi) @@ -559,18 +556,21 @@ static inline int reserved_sections(struct f2fs_sb_info *sbi) } static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi, - unsigned int node_blocks, unsigned int dent_blocks) + unsigned int node_blocks, unsigned int data_blocks, + unsigned int dent_blocks) { - unsigned segno, left_blocks; + unsigned int segno, left_blocks, blocks; int i; - /* check current node sections in the worst case. */ - for (i = CURSEG_HOT_NODE; i <= CURSEG_COLD_NODE; i++) { + /* check current data/node sections in the worst case. */ + for (i = CURSEG_HOT_DATA; i < NR_PERSISTENT_LOG; i++) { segno = CURSEG_I(sbi, i)->segno; left_blocks = CAP_BLKS_PER_SEC(sbi) - get_ckpt_valid_blocks(sbi, segno, true); - if (node_blocks > left_blocks) + + blocks = i <= CURSEG_COLD_DATA ? data_blocks : node_blocks; + if (blocks > left_blocks) return false; } @@ -584,8 +584,9 @@ static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi, } /* - * calculate needed sections for dirty node/dentry - * and call has_curseg_enough_space + * calculate needed sections for dirty node/dentry and call + * has_curseg_enough_space, please note that, it needs to account + * dirty data as well in lfs mode when checkpoint is disabled. */ static inline void __get_secs_required(struct f2fs_sb_info *sbi, unsigned int *lower_p, unsigned int *upper_p, bool *curseg_p) @@ -594,19 +595,30 @@ static inline void __get_secs_required(struct f2fs_sb_info *sbi, get_pages(sbi, F2FS_DIRTY_DENTS) + get_pages(sbi, F2FS_DIRTY_IMETA); unsigned int total_dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS); + unsigned int total_data_blocks = 0; unsigned int node_secs = total_node_blocks / CAP_BLKS_PER_SEC(sbi); unsigned int dent_secs = total_dent_blocks / CAP_BLKS_PER_SEC(sbi); + unsigned int data_secs = 0; unsigned int node_blocks = total_node_blocks % CAP_BLKS_PER_SEC(sbi); unsigned int dent_blocks = total_dent_blocks % CAP_BLKS_PER_SEC(sbi); + unsigned int data_blocks = 0; + + if (f2fs_lfs_mode(sbi) && + unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + total_data_blocks = get_pages(sbi, F2FS_DIRTY_DATA); + data_secs = total_data_blocks / CAP_BLKS_PER_SEC(sbi); + data_blocks = total_data_blocks % CAP_BLKS_PER_SEC(sbi); + } if (lower_p) - *lower_p = node_secs + dent_secs; + *lower_p = node_secs + dent_secs + data_secs; if (upper_p) *upper_p = node_secs + dent_secs + - (node_blocks ? 1 : 0) + (dent_blocks ? 1 : 0); + (node_blocks ? 1 : 0) + (dent_blocks ? 1 : 0) + + (data_blocks ? 1 : 0); if (curseg_p) *curseg_p = has_curseg_enough_space(sbi, - node_blocks, dent_blocks); + node_blocks, data_blocks, dent_blocks); } static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, @@ -637,12 +649,30 @@ static inline bool has_enough_free_secs(struct f2fs_sb_info *sbi, return !has_not_enough_free_secs(sbi, freed, needed); } +static inline bool has_enough_free_blks(struct f2fs_sb_info *sbi) +{ + unsigned int total_free_blocks = 0; + unsigned int avail_user_block_count; + + spin_lock(&sbi->stat_lock); + + avail_user_block_count = get_available_block_count(sbi, NULL, true); + total_free_blocks = avail_user_block_count - (unsigned int)valid_user_blocks(sbi); + + spin_unlock(&sbi->stat_lock); + + return total_free_blocks > 0; +} + static inline bool f2fs_is_checkpoint_ready(struct f2fs_sb_info *sbi) { if (likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) return true; if (likely(has_enough_free_secs(sbi, 0, 0))) return true; + if (!f2fs_lfs_mode(sbi) && + likely(has_enough_free_blks(sbi))) + return true; return false; } @@ -957,13 +987,3 @@ wake_up: dcc->discard_wake = true; wake_up_interruptible_all(&dcc->discard_wait_queue); } - -static inline unsigned int first_zoned_segno(struct f2fs_sb_info *sbi) -{ - int devi; - - for (devi = 0; devi < sbi->s_ndevs; devi++) - if (bdev_is_zoned(FDEV(devi).bdev)) - return GET_SEGNO(sbi, FDEV(devi).start_blk); - return 0; -} diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 87ab5696bd48..fc7d463dee15 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -150,6 +150,8 @@ enum { Opt_mode, Opt_fault_injection, Opt_fault_type, + Opt_lazytime, + Opt_nolazytime, Opt_quota, Opt_noquota, Opt_usrquota, @@ -226,6 +228,8 @@ static match_table_t f2fs_tokens = { {Opt_mode, "mode=%s"}, {Opt_fault_injection, "fault_injection=%u"}, {Opt_fault_type, "fault_type=%u"}, + {Opt_lazytime, "lazytime"}, + {Opt_nolazytime, "nolazytime"}, {Opt_quota, "quota"}, {Opt_noquota, "noquota"}, {Opt_usrquota, "usrquota"}, @@ -834,6 +838,10 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) set_opt(sbi, READ_EXTENT_CACHE); break; case Opt_noextent_cache: + if (F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_DEVICE_ALIAS)) { + f2fs_err(sbi, "device aliasing requires extent cache"); + return -EINVAL; + } clear_opt(sbi, READ_EXTENT_CACHE); break; case Opt_noinline_data: @@ -918,6 +926,12 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) f2fs_info(sbi, "fault_type options not supported"); break; #endif + case Opt_lazytime: + sb->s_flags |= SB_LAZYTIME; + break; + case Opt_nolazytime: + sb->s_flags &= ~SB_LAZYTIME; + break; #ifdef CONFIG_QUOTA case Opt_quota: case Opt_usrquota: @@ -1158,7 +1172,11 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) break; } - strcpy(ext[ext_cnt], name); + ret = strscpy(ext[ext_cnt], name); + if (ret < 0) { + kfree(name); + return ret; + } F2FS_OPTION(sbi).compress_ext_cnt++; kfree(name); break; @@ -1187,7 +1205,11 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) break; } - strcpy(noext[noext_cnt], name); + ret = strscpy(noext[noext_cnt], name); + if (ret < 0) { + kfree(name); + return ret; + } F2FS_OPTION(sbi).nocompress_ext_cnt++; kfree(name); break; @@ -1738,6 +1760,18 @@ static int f2fs_freeze(struct super_block *sb) static int f2fs_unfreeze(struct super_block *sb) { + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + /* + * It will update discard_max_bytes of mounted lvm device to zero + * after creating snapshot on this lvm device, let's drop all + * remained discards. + * We don't need to disable real-time discard because discard_max_bytes + * will recover after removal of snapshot. + */ + if (test_opt(sbi, DISCARD) && !f2fs_hw_support_discard(sbi)) + f2fs_issue_discard_timeout(sbi); + clear_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING); return 0; } @@ -2474,6 +2508,7 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) } } + adjust_unusable_cap_perc(sbi); if (enable_checkpoint == !!test_opt(sbi, DISABLE_CHECKPOINT)) { if (test_opt(sbi, DISABLE_CHECKPOINT)) { err = f2fs_disable_checkpoint(sbi); @@ -2518,7 +2553,6 @@ skip: (test_opt(sbi, POSIX_ACL) ? SB_POSIXACL : 0); limit_reserve_root(sbi); - adjust_unusable_cap_perc(sbi); *flags = (*flags & ~SB_LAZYTIME) | (sb->s_flags & SB_LAZYTIME); return 0; restore_checkpoint: @@ -3322,7 +3356,7 @@ loff_t max_file_blocks(struct inode *inode) * fit within U32_MAX + 1 data units. */ - result = min(result, F2FS_BYTES_TO_BLK(((loff_t)U32_MAX + 1) * 4096)); + result = umin(result, F2FS_BYTES_TO_BLK(((loff_t)U32_MAX + 1) * 4096)); return result; } @@ -4155,8 +4189,7 @@ static bool system_going_down(void) || system_state == SYSTEM_RESTART; } -void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason, - bool irq_context) +void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason) { struct super_block *sb = sbi->sb; bool shutdown = reason == STOP_CP_REASON_SHUTDOWN; @@ -4168,10 +4201,12 @@ void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason, if (!f2fs_hw_is_readonly(sbi)) { save_stop_reason(sbi, reason); - if (irq_context && !shutdown) - schedule_work(&sbi->s_error_work); - else - f2fs_record_stop_reason(sbi); + /* + * always create an asynchronous task to record stop_reason + * in order to avoid potential deadlock when running into + * f2fs_record_stop_reason() synchronously. + */ + schedule_work(&sbi->s_error_work); } /* @@ -4217,6 +4252,16 @@ static void f2fs_record_error_work(struct work_struct *work) f2fs_record_stop_reason(sbi); } +static inline unsigned int get_first_zoned_segno(struct f2fs_sb_info *sbi) +{ + int devi; + + for (devi = 0; devi < sbi->s_ndevs; devi++) + if (bdev_is_zoned(FDEV(devi).bdev)) + return GET_SEGNO(sbi, FDEV(devi).start_blk); + return 0; +} + static int f2fs_scan_devices(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); @@ -4617,6 +4662,9 @@ try_onemore: /* For write statistics */ sbi->sectors_written_start = f2fs_get_sectors_written(sbi); + /* get segno of first zoned block device */ + sbi->first_zoned_segno = get_first_zoned_segno(sbi); + /* Read accumulated write IO statistics if exists */ seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE); if (__exist_node_summaries(sbi)) @@ -4738,26 +4786,23 @@ try_onemore: reset_checkpoint: /* * If the f2fs is not readonly and fsync data recovery succeeds, - * check zoned block devices' write pointer consistency. + * write pointer consistency of cursegs and other zones are already + * checked and fixed during recovery. However, if recovery fails, + * write pointers are left untouched, and retry-mount should check + * them here. */ - if (f2fs_sb_has_blkzoned(sbi) && !f2fs_readonly(sb)) { - int err2; - - f2fs_notice(sbi, "Checking entire write pointers"); - err2 = f2fs_check_write_pointer(sbi); - if (err2) - err = err2; - } + if (skip_recovery) + err = f2fs_check_and_fix_write_pointer(sbi); if (err) goto free_meta; + /* f2fs_recover_fsync_data() cleared this already */ + clear_sbi_flag(sbi, SBI_POR_DOING); + err = f2fs_init_inmem_curseg(sbi); if (err) goto sync_free_meta; - /* f2fs_recover_fsync_data() cleared this already */ - clear_sbi_flag(sbi, SBI_POR_DOING); - if (test_opt(sbi, DISABLE_CHECKPOINT)) { err = f2fs_disable_checkpoint(sbi); if (err) @@ -4991,9 +5036,6 @@ static int __init init_f2fs_fs(void) err = f2fs_init_shrinker(); if (err) goto free_sysfs; - err = register_filesystem(&f2fs_fs_type); - if (err) - goto free_shrinker; f2fs_create_root_stats(); err = f2fs_init_post_read_processing(); if (err) @@ -5016,7 +5058,12 @@ static int __init init_f2fs_fs(void) err = f2fs_create_casefold_cache(); if (err) goto free_compress_cache; + err = register_filesystem(&f2fs_fs_type); + if (err) + goto free_casefold_cache; return 0; +free_casefold_cache: + f2fs_destroy_casefold_cache(); free_compress_cache: f2fs_destroy_compress_cache(); free_compress_mempool: @@ -5031,8 +5078,6 @@ free_post_read: f2fs_destroy_post_read_processing(); free_root_stats: f2fs_destroy_root_stats(); - unregister_filesystem(&f2fs_fs_type); -free_shrinker: f2fs_exit_shrinker(); free_sysfs: f2fs_exit_sysfs(); @@ -5056,6 +5101,7 @@ fail: static void __exit exit_f2fs_fs(void) { + unregister_filesystem(&f2fs_fs_type); f2fs_destroy_casefold_cache(); f2fs_destroy_compress_cache(); f2fs_destroy_compress_mempool(); @@ -5064,7 +5110,6 @@ static void __exit exit_f2fs_fs(void) f2fs_destroy_iostat_processing(); f2fs_destroy_post_read_processing(); f2fs_destroy_root_stats(); - unregister_filesystem(&f2fs_fs_type); f2fs_exit_shrinker(); f2fs_exit_sysfs(); f2fs_destroy_garbage_collection_cache(); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index c56e8c873935..6b99dc49f776 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -501,9 +501,7 @@ out: if (a->struct_type == RESERVED_BLOCKS) { spin_lock(&sbi->stat_lock); if (t > (unsigned long)(sbi->user_block_count - - F2FS_OPTION(sbi).root_reserved_blocks - - SEGS_TO_BLKS(sbi, - SM_I(sbi)->additional_reserved_segments))) { + F2FS_OPTION(sbi).root_reserved_blocks)) { spin_unlock(&sbi->stat_lock); return -EINVAL; } @@ -789,6 +787,13 @@ out: return count; } + if (!strcmp(a->attr.name, "max_read_extent_count")) { + if (t > UINT_MAX) + return -EINVAL; + *ui = (unsigned int)t; + return count; + } + if (!strcmp(a->attr.name, "ipu_policy")) { if (t >= BIT(F2FS_IPU_MAX)) return -EINVAL; @@ -1054,6 +1059,8 @@ F2FS_SBI_GENERAL_RW_ATTR(revoked_atomic_block); F2FS_SBI_GENERAL_RW_ATTR(hot_data_age_threshold); F2FS_SBI_GENERAL_RW_ATTR(warm_data_age_threshold); F2FS_SBI_GENERAL_RW_ATTR(last_age_weight); +/* read extent cache */ +F2FS_SBI_GENERAL_RW_ATTR(max_read_extent_count); #ifdef CONFIG_BLK_DEV_ZONED F2FS_SBI_GENERAL_RO_ATTR(unusable_blocks_per_sec); F2FS_SBI_GENERAL_RW_ATTR(blkzone_alloc_policy); @@ -1244,6 +1251,7 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(hot_data_age_threshold), ATTR_LIST(warm_data_age_threshold), ATTR_LIST(last_age_weight), + ATTR_LIST(max_read_extent_count), NULL, }; ATTRIBUTE_GROUPS(f2fs); @@ -1313,6 +1321,7 @@ F2FS_SB_FEATURE_RO_ATTR(sb_checksum, SB_CHKSUM); F2FS_SB_FEATURE_RO_ATTR(casefold, CASEFOLD); F2FS_SB_FEATURE_RO_ATTR(compression, COMPRESSION); F2FS_SB_FEATURE_RO_ATTR(readonly, RO); +F2FS_SB_FEATURE_RO_ATTR(device_alias, DEVICE_ALIAS); static struct attribute *f2fs_sb_feat_attrs[] = { ATTR_LIST(sb_encryption), @@ -1329,6 +1338,7 @@ static struct attribute *f2fs_sb_feat_attrs[] = { ATTR_LIST(sb_casefold), ATTR_LIST(sb_compression), ATTR_LIST(sb_readonly), + ATTR_LIST(sb_device_alias), NULL, }; ATTRIBUTE_GROUPS(f2fs_sb_feat); diff --git a/fs/fhandle.c b/fs/fhandle.c index 5f801139358e..ec9145047dfc 100644 --- a/fs/fhandle.c +++ b/fs/fhandle.c @@ -31,6 +31,14 @@ static long do_sys_name_to_handle(const struct path *path, if (!exportfs_can_encode_fh(path->dentry->d_sb->s_export_op, fh_flags)) return -EOPNOTSUPP; + /* + * A request to encode a connectable handle for a disconnected dentry + * is unexpected since AT_EMPTY_PATH is not allowed. + */ + if (fh_flags & EXPORT_FH_CONNECTABLE && + WARN_ON(path->dentry->d_flags & DCACHE_DISCONNECTED)) + return -EINVAL; + if (copy_from_user(&f_handle, ufh, sizeof(struct file_handle))) return -EFAULT; @@ -45,7 +53,7 @@ static long do_sys_name_to_handle(const struct path *path, /* convert handle size to multiple of sizeof(u32) */ handle_dwords = f_handle.handle_bytes >> 2; - /* we ask for a non connectable maybe decodeable file handle */ + /* Encode a possibly decodeable/connectable file handle */ retval = exportfs_encode_fh(path->dentry, (struct fid *)handle->f_handle, &handle_dwords, fh_flags); @@ -67,8 +75,23 @@ static long do_sys_name_to_handle(const struct path *path, * non variable part of the file_handle */ handle_bytes = 0; - } else + } else { + /* + * When asked to encode a connectable file handle, encode this + * property in the file handle itself, so that we later know + * how to decode it. + * For sanity, also encode in the file handle if the encoded + * object is a directory and verify this during decode, because + * decoding directory file handles is quite different than + * decoding connectable non-directory file handles. + */ + if (fh_flags & EXPORT_FH_CONNECTABLE) { + handle->handle_type |= FILEID_IS_CONNECTABLE; + if (d_is_dir(path->dentry)) + fh_flags |= FILEID_IS_DIR; + } retval = 0; + } /* copy the mount id */ if (unique_mntid) { if (put_user(real_mount(path->mnt)->mnt_id_unique, @@ -109,15 +132,30 @@ SYSCALL_DEFINE5(name_to_handle_at, int, dfd, const char __user *, name, { struct path path; int lookup_flags; - int fh_flags; + int fh_flags = 0; int err; if (flag & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH | AT_HANDLE_FID | - AT_HANDLE_MNT_ID_UNIQUE)) + AT_HANDLE_MNT_ID_UNIQUE | AT_HANDLE_CONNECTABLE)) + return -EINVAL; + + /* + * AT_HANDLE_FID means there is no intention to decode file handle + * AT_HANDLE_CONNECTABLE means there is an intention to decode a + * connected fd (with known path), so these flags are conflicting. + * AT_EMPTY_PATH could be used along with a dfd that refers to a + * disconnected non-directory, which cannot be used to encode a + * connectable file handle, because its parent is unknown. + */ + if (flag & AT_HANDLE_CONNECTABLE && + flag & (AT_HANDLE_FID | AT_EMPTY_PATH)) return -EINVAL; + else if (flag & AT_HANDLE_FID) + fh_flags |= EXPORT_FH_FID; + else if (flag & AT_HANDLE_CONNECTABLE) + fh_flags |= EXPORT_FH_CONNECTABLE; lookup_flags = (flag & AT_SYMLINK_FOLLOW) ? LOOKUP_FOLLOW : 0; - fh_flags = (flag & AT_HANDLE_FID) ? EXPORT_FH_FID : 0; if (flag & AT_EMPTY_PATH) lookup_flags |= LOOKUP_EMPTY; err = user_path_at(dfd, name, lookup_flags, &path); @@ -207,7 +245,13 @@ static int vfs_dentry_acceptable(void *context, struct dentry *dentry) if (!(ctx->flags & HANDLE_CHECK_SUBTREE) || d == root) retval = 1; - WARN_ON_ONCE(d != root && d != root->d_sb->s_root); + /* + * exportfs_decode_fh_raw() does not call acceptable() callback with + * a disconnected directory dentry, so we should have reached either + * mount fd directory or sb root. + */ + if (ctx->fh_flags & EXPORT_FH_DIR_ONLY) + WARN_ON_ONCE(d != root && d != root->d_sb->s_root); dput(d); return retval; } @@ -306,6 +350,12 @@ static int handle_to_path(int mountdirfd, struct file_handle __user *ufh, retval = -EINVAL; goto out_path; } + if (f_handle.handle_type < 0 || + FILEID_USER_FLAGS(f_handle.handle_type) & ~FILEID_VALID_USER_FLAGS) { + retval = -EINVAL; + goto out_path; + } + handle = kmalloc(struct_size(handle, f_handle, f_handle.handle_bytes), GFP_KERNEL); if (!handle) { @@ -321,6 +371,19 @@ static int handle_to_path(int mountdirfd, struct file_handle __user *ufh, goto out_handle; } + /* + * If handle was encoded with AT_HANDLE_CONNECTABLE, verify that we + * are decoding an fd with connected path, which is accessible from + * the mount fd path. + */ + if (f_handle.handle_type & FILEID_IS_CONNECTABLE) { + ctx.fh_flags |= EXPORT_FH_CONNECTABLE; + ctx.flags |= HANDLE_CHECK_SUBTREE; + } + if (f_handle.handle_type & FILEID_IS_DIR) + ctx.fh_flags |= EXPORT_FH_DIR_ONLY; + /* Filesystem code should not be exposed to user flags */ + handle->handle_type &= ~FILEID_USER_FLAGS_MASK; retval = do_handle_to_path(handle, path, &ctx); out_handle: diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index ce0ff7a9007b..2c372180d631 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -14,5 +14,6 @@ fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o +fuse-$(CONFIG_SYSCTL) += sysctl.o virtiofs-y := virtio_fs.o diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 0b2da7b7e2ad..b39844d75a80 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -303,8 +303,8 @@ struct cuse_init_args { struct fuse_args_pages ap; struct cuse_init_in in; struct cuse_init_out out; - struct page *page; - struct fuse_page_desc desc; + struct folio *folio; + struct fuse_folio_desc desc; }; /** @@ -326,7 +326,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, struct fuse_args_pages *ap = &ia->ap; struct cuse_conn *cc = fc_to_cc(fc), *pos; struct cuse_init_out *arg = &ia->out; - struct page *page = ap->pages[0]; + struct folio *folio = ap->folios[0]; struct cuse_devinfo devinfo = { }; struct device *dev; struct cdev *cdev; @@ -343,7 +343,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, /* parse init reply */ cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL; - rc = cuse_parse_devinfo(page_address(page), ap->args.out_args[1].size, + rc = cuse_parse_devinfo(folio_address(folio), ap->args.out_args[1].size, &devinfo); if (rc) goto err; @@ -411,7 +411,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm, kobject_uevent(&dev->kobj, KOBJ_ADD); out: kfree(ia); - __free_page(page); + folio_put(folio); return; err_cdev: @@ -429,7 +429,7 @@ err: static int cuse_send_init(struct cuse_conn *cc) { int rc; - struct page *page; + struct folio *folio; struct fuse_mount *fm = &cc->fm; struct cuse_init_args *ia; struct fuse_args_pages *ap; @@ -437,13 +437,14 @@ static int cuse_send_init(struct cuse_conn *cc) BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE); rc = -ENOMEM; - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) + + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + if (!folio) goto err; ia = kzalloc(sizeof(*ia), GFP_KERNEL); if (!ia) - goto err_free_page; + goto err_free_folio; ap = &ia->ap; ia->in.major = FUSE_KERNEL_VERSION; @@ -459,18 +460,18 @@ static int cuse_send_init(struct cuse_conn *cc) ap->args.out_args[1].size = CUSE_INIT_INFO_MAX; ap->args.out_argvar = true; ap->args.out_pages = true; - ap->num_pages = 1; - ap->pages = &ia->page; + ap->num_folios = 1; + ap->folios = &ia->folio; ap->descs = &ia->desc; - ia->page = page; + ia->folio = folio; ia->desc.length = ap->args.out_args[1].size; ap->args.end = cuse_process_init_reply; rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (rc) { kfree(ia); -err_free_page: - __free_page(page); +err_free_folio: + folio_put(folio); } err: return rc; diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c index 12ef91d170bb..9abbc2f2894f 100644 --- a/fs/fuse/dax.c +++ b/fs/fuse/dax.c @@ -774,16 +774,6 @@ out: return ret; } -static int fuse_dax_writepages(struct address_space *mapping, - struct writeback_control *wbc) -{ - - struct inode *inode = mapping->host; - struct fuse_conn *fc = get_fuse_conn(inode); - - return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc); -} - static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order, bool write) { @@ -1323,7 +1313,6 @@ bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) } static const struct address_space_operations fuse_dax_file_aops = { - .writepages = fuse_dax_writepages, .direct_IO = noop_direct_IO, .dirty_folio = noop_dirty_folio, }; diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 0723c6344b20..27ccae63495d 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -1028,17 +1028,27 @@ static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, struct fuse_req *req = cs->req; struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); - - for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) { + for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) { int err; unsigned int offset = ap->descs[i].offset; unsigned int count = min(nbytes, ap->descs[i].length); + struct page *orig, *pagep; + + orig = pagep = &ap->folios[i]->page; - err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing); + err = fuse_copy_page(cs, &pagep, offset, count, zeroing); if (err) return err; nbytes -= count; + + /* + * fuse_copy_page may have moved a page from a pipe instead of + * copying into our given page, so update the folios if it was + * replaced. + */ + if (pagep != orig) + ap->folios[i] = page_folio(pagep); } return 0; } @@ -1654,24 +1664,25 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, num = outarg.size; while (num) { + struct folio *folio; struct page *page; unsigned int this_num; - err = -ENOMEM; - page = find_or_create_page(mapping, index, - mapping_gfp_mask(mapping)); - if (!page) + folio = filemap_grab_folio(mapping, index); + err = PTR_ERR(folio); + if (IS_ERR(folio)) goto out_iput; - this_num = min_t(unsigned, num, PAGE_SIZE - offset); + page = &folio->page; + this_num = min_t(unsigned, num, folio_size(folio) - offset); err = fuse_copy_page(cs, &page, offset, this_num, 0); - if (!PageUptodate(page) && !err && offset == 0 && - (this_num == PAGE_SIZE || file_size == end)) { - zero_user_segment(page, this_num, PAGE_SIZE); - SetPageUptodate(page); + if (!folio_test_uptodate(folio) && !err && offset == 0 && + (this_num == folio_size(folio) || file_size == end)) { + folio_zero_segment(folio, this_num, folio_size(folio)); + folio_mark_uptodate(folio); } - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); if (err) goto out_iput; @@ -1703,7 +1714,7 @@ static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, struct fuse_retrieve_args *ra = container_of(args, typeof(*ra), ap.args); - release_pages(ra->ap.pages, ra->ap.num_pages); + release_pages(ra->ap.folios, ra->ap.num_folios); kfree(ra); } @@ -1717,7 +1728,7 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, unsigned int num; unsigned int offset; size_t total_len = 0; - unsigned int num_pages; + unsigned int num_pages, cur_pages = 0; struct fuse_conn *fc = fm->fc; struct fuse_retrieve_args *ra; size_t args_size = sizeof(*ra); @@ -1736,15 +1747,15 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; num_pages = min(num_pages, fc->max_pages); - args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0])); + args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); ra = kzalloc(args_size, GFP_KERNEL); if (!ra) return -ENOMEM; ap = &ra->ap; - ap->pages = (void *) (ra + 1); - ap->descs = (void *) (ap->pages + num_pages); + ap->folios = (void *) (ra + 1); + ap->descs = (void *) (ap->folios + num_pages); args = &ap->args; args->nodeid = outarg->nodeid; @@ -1755,19 +1766,20 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, index = outarg->offset >> PAGE_SHIFT; - while (num && ap->num_pages < num_pages) { - struct page *page; + while (num && cur_pages < num_pages) { + struct folio *folio; unsigned int this_num; - page = find_get_page(mapping, index); - if (!page) + folio = filemap_get_folio(mapping, index); + if (IS_ERR(folio)) break; this_num = min_t(unsigned, num, PAGE_SIZE - offset); - ap->pages[ap->num_pages] = page; - ap->descs[ap->num_pages].offset = offset; - ap->descs[ap->num_pages].length = this_num; - ap->num_pages++; + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].offset = offset; + ap->descs[ap->num_folios].length = this_num; + ap->num_folios++; + cur_pages++; offset = 0; num -= this_num; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 54104dd48af7..494ac372ace0 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -366,7 +366,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name struct fuse_mount *fm = get_fuse_mount_super(sb); FUSE_ARGS(args); struct fuse_forget_link *forget; - u64 attr_version; + u64 attr_version, evict_ctr; int err; *inode = NULL; @@ -381,6 +381,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name goto out; attr_version = fuse_get_attr_version(fm->fc); + evict_ctr = fuse_get_evict_ctr(fm->fc); fuse_lookup_init(fm->fc, &args, nodeid, name, outarg); err = fuse_simple_request(fm, &args); @@ -398,7 +399,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, &outarg->attr, ATTR_TIMEOUT(outarg), - attr_version); + attr_version, evict_ctr); err = -ENOMEM; if (!*inode) { fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1); @@ -691,7 +692,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, ff->nodeid = outentry.nodeid; ff->open_flags = outopenp->open_flags; inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation, - &outentry.attr, ATTR_TIMEOUT(&outentry), 0); + &outentry.attr, ATTR_TIMEOUT(&outentry), 0, 0); if (!inode) { flags &= ~(O_CREAT | O_EXCL | O_TRUNC); fuse_sync_release(NULL, ff, flags); @@ -822,7 +823,7 @@ static int create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, goto out_put_forget_req; inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, - &outarg.attr, ATTR_TIMEOUT(&outarg), 0); + &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0); if (!inode) { fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); return -ENOMEM; @@ -1585,13 +1586,13 @@ static int fuse_permission(struct mnt_idmap *idmap, return err; } -static int fuse_readlink_page(struct inode *inode, struct page *page) +static int fuse_readlink_page(struct inode *inode, struct folio *folio) { struct fuse_mount *fm = get_fuse_mount(inode); - struct fuse_page_desc desc = { .length = PAGE_SIZE - 1 }; + struct fuse_folio_desc desc = { .length = PAGE_SIZE - 1 }; struct fuse_args_pages ap = { - .num_pages = 1, - .pages = &page, + .num_folios = 1, + .folios = &folio, .descs = &desc, }; char *link; @@ -1614,7 +1615,7 @@ static int fuse_readlink_page(struct inode *inode, struct page *page) if (WARN_ON(res >= PAGE_SIZE)) return -EIO; - link = page_address(page); + link = folio_address(folio); link[res] = '\0'; return 0; @@ -1624,7 +1625,7 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *callback) { struct fuse_conn *fc = get_fuse_conn(inode); - struct page *page; + struct folio *folio; int err; err = -EIO; @@ -1638,20 +1639,20 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, if (!dentry) goto out_err; - page = alloc_page(GFP_KERNEL); + folio = folio_alloc(GFP_KERNEL, 0); err = -ENOMEM; - if (!page) + if (!folio) goto out_err; - err = fuse_readlink_page(inode, page); + err = fuse_readlink_page(inode, folio); if (err) { - __free_page(page); + folio_put(folio); goto out_err; } - set_delayed_call(callback, page_put_link, page); + set_delayed_call(callback, page_put_link, &folio->page); - return page_address(page); + return folio_address(folio); out_err: return ERR_PTR(err); @@ -2028,7 +2029,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, fuse_change_attributes_common(inode, &outarg.attr, NULL, ATTR_TIMEOUT(&outarg), - fuse_get_cache_mask(inode)); + fuse_get_cache_mask(inode), 0); oldsize = inode->i_size; /* see the comment in fuse_change_attributes() */ if (!is_wb || is_truncate) @@ -2231,7 +2232,7 @@ void fuse_init_dir(struct inode *inode) static int fuse_symlink_read_folio(struct file *null, struct folio *folio) { - int err = fuse_readlink_page(folio->mapping->host, &folio->page); + int err = fuse_readlink_page(folio->mapping->host, folio); if (!err) folio_mark_uptodate(folio); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index dafdf766b1d5..88d0946b5bc9 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -436,7 +436,7 @@ static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi, wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry); WARN_ON(get_fuse_inode(wpa->inode) != fi); curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT; - if (idx_from >= curr_index + wpa->ia.ap.num_pages) + if (idx_from >= curr_index + wpa->ia.ap.num_folios) n = n->rb_right; else if (idx_to < curr_index) n = n->rb_left; @@ -483,6 +483,21 @@ static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); } +static inline bool fuse_folio_is_writeback(struct inode *inode, + struct folio *folio) +{ + pgoff_t last = folio_next_index(folio) - 1; + return fuse_range_is_writeback(inode, folio_index(folio), last); +} + +static void fuse_wait_on_folio_writeback(struct inode *inode, + struct folio *folio) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + wait_event(fi->page_waitq, !fuse_folio_is_writeback(inode, folio)); +} + /* * Wait for all pending writepages on the inode to finish. * @@ -645,17 +660,20 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->out_args[0].size = count; } -static void fuse_release_user_pages(struct fuse_args_pages *ap, +static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, bool should_dirty) { unsigned int i; - for (i = 0; i < ap->num_pages; i++) { + for (i = 0; i < ap->num_folios; i++) { if (should_dirty) - set_page_dirty_lock(ap->pages[i]); + folio_mark_dirty_lock(ap->folios[i]); if (ap->args.is_pinned) - unpin_user_page(ap->pages[i]); + unpin_folio(ap->folios[i]); } + + if (nres > 0 && ap->args.invalidate_vmap) + invalidate_kernel_vmap_range(ap->args.vmap_base, nres); } static void fuse_io_release(struct kref *kref) @@ -725,16 +743,16 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) } static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, - unsigned int npages) + unsigned int nfolios) { struct fuse_io_args *ia; ia = kzalloc(sizeof(*ia), GFP_KERNEL); if (ia) { ia->io = io; - ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL, - &ia->ap.descs); - if (!ia->ap.pages) { + ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL, + &ia->ap.descs); + if (!ia->ap.folios) { kfree(ia); ia = NULL; } @@ -744,7 +762,7 @@ static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, static void fuse_io_free(struct fuse_io_args *ia) { - kfree(ia->ap.pages); + kfree(ia->ap.folios); kfree(ia); } @@ -754,25 +772,29 @@ static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); struct fuse_io_priv *io = ia->io; ssize_t pos = -1; - - fuse_release_user_pages(&ia->ap, io->should_dirty); + size_t nres; if (err) { /* Nothing */ } else if (io->write) { if (ia->write.out.size > ia->write.in.size) { err = -EIO; - } else if (ia->write.in.size != ia->write.out.size) { - pos = ia->write.in.offset - io->offset + - ia->write.out.size; + } else { + nres = ia->write.out.size; + if (ia->write.in.size != ia->write.out.size) + pos = ia->write.in.offset - io->offset + + ia->write.out.size; } } else { u32 outsize = args->out_args[0].size; + nres = outsize; if (ia->read.in.size != outsize) pos = ia->read.in.offset - io->offset + outsize; } + fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty); + fuse_aio_complete(io, err, pos); fuse_io_free(ia); } @@ -843,33 +865,33 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, * reached the client fs yet. So the hole is not present there. */ if (!fc->writeback_cache) { - loff_t pos = page_offset(ap->pages[0]) + num_read; + loff_t pos = folio_pos(ap->folios[0]) + num_read; fuse_read_update_size(inode, pos, attr_ver); } } -static int fuse_do_readpage(struct file *file, struct page *page) +static int fuse_do_readfolio(struct file *file, struct folio *folio) { - struct inode *inode = page->mapping->host; + struct inode *inode = folio->mapping->host; struct fuse_mount *fm = get_fuse_mount(inode); - loff_t pos = page_offset(page); - struct fuse_page_desc desc = { .length = PAGE_SIZE }; + loff_t pos = folio_pos(folio); + struct fuse_folio_desc desc = { .length = PAGE_SIZE }; struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, - .ap.num_pages = 1, - .ap.pages = &page, + .ap.num_folios = 1, + .ap.folios = &folio, .ap.descs = &desc, }; ssize_t res; u64 attr_ver; /* - * Page writeback can extend beyond the lifetime of the - * page-cache page, so make sure we read a properly synced - * page. + * With the temporary pages that are used to complete writeback, we can + * have writeback that extends beyond the lifetime of the folio. So + * make sure we read a properly synced folio. */ - fuse_wait_on_page_writeback(inode, page->index); + fuse_wait_on_folio_writeback(inode, folio); attr_ver = fuse_get_attr_version(fm->fc); @@ -887,25 +909,24 @@ static int fuse_do_readpage(struct file *file, struct page *page) if (res < desc.length) fuse_short_read(inode, attr_ver, res, &ia.ap); - SetPageUptodate(page); + folio_mark_uptodate(folio); return 0; } static int fuse_read_folio(struct file *file, struct folio *folio) { - struct page *page = &folio->page; - struct inode *inode = page->mapping->host; + struct inode *inode = folio->mapping->host; int err; err = -EIO; if (fuse_is_bad(inode)) goto out; - err = fuse_do_readpage(file, page); + err = fuse_do_readfolio(file, folio); fuse_invalidate_atime(inode); out: - unlock_page(page); + folio_unlock(folio); return err; } @@ -919,8 +940,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, size_t num_read = args->out_args[0].size; struct address_space *mapping = NULL; - for (i = 0; mapping == NULL && i < ap->num_pages; i++) - mapping = ap->pages[i]->mapping; + for (i = 0; mapping == NULL && i < ap->num_folios; i++) + mapping = ap->folios[i]->mapping; if (mapping) { struct inode *inode = mapping->host; @@ -934,12 +955,8 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, fuse_invalidate_atime(inode); } - for (i = 0; i < ap->num_pages; i++) { - struct folio *folio = page_folio(ap->pages[i]); - - folio_end_read(folio, !err); - folio_put(folio); - } + for (i = 0; i < ap->num_folios; i++) + folio_end_read(ap->folios[i], !err); if (ia->ff) fuse_file_put(ia->ff, false); @@ -951,8 +968,9 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; struct fuse_args_pages *ap = &ia->ap; - loff_t pos = page_offset(ap->pages[0]); - size_t count = ap->num_pages << PAGE_SHIFT; + loff_t pos = folio_pos(ap->folios[0]); + /* Currently, all folios in FUSE are one page */ + size_t count = ap->num_folios << PAGE_SHIFT; ssize_t res; int err; @@ -963,7 +981,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { count--; - ap->descs[ap->num_pages - 1].length--; + ap->descs[ap->num_folios - 1].length--; } WARN_ON((loff_t) (pos + count) < 0); @@ -985,18 +1003,36 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) static void fuse_readahead(struct readahead_control *rac) { struct inode *inode = rac->mapping->host; + struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); - unsigned int i, max_pages, nr_pages = 0; + unsigned int max_pages, nr_pages; + pgoff_t first = readahead_index(rac); + pgoff_t last = first + readahead_count(rac) - 1; if (fuse_is_bad(inode)) return; + wait_event(fi->page_waitq, !fuse_range_is_writeback(inode, first, last)); + max_pages = min_t(unsigned int, fc->max_pages, fc->max_read / PAGE_SIZE); - for (;;) { + /* + * This is only accurate the first time through, since readahead_folio() + * doesn't update readahead_count() from the previous folio until the + * next call. Grab nr_pages here so we know how many pages we're going + * to have to process. This means that we will exit here with + * readahead_count() == folio_nr_pages(last_folio), but we will have + * consumed all of the folios, and read_pages() will call + * readahead_folio() again which will clean up the rac. + */ + nr_pages = readahead_count(rac); + + while (nr_pages) { struct fuse_io_args *ia; struct fuse_args_pages *ap; + struct folio *folio; + unsigned cur_pages = min(max_pages, nr_pages); if (fc->num_background >= fc->congestion_threshold && rac->ra->async_size >= readahead_count(rac)) @@ -1006,23 +1042,19 @@ static void fuse_readahead(struct readahead_control *rac) */ break; - nr_pages = readahead_count(rac) - nr_pages; - if (nr_pages > max_pages) - nr_pages = max_pages; - if (nr_pages == 0) - break; - ia = fuse_io_alloc(NULL, nr_pages); + ia = fuse_io_alloc(NULL, cur_pages); if (!ia) return; ap = &ia->ap; - nr_pages = __readahead_batch(rac, ap->pages, nr_pages); - for (i = 0; i < nr_pages; i++) { - fuse_wait_on_page_writeback(inode, - readahead_index(rac) + i); - ap->descs[i].length = PAGE_SIZE; + + while (ap->num_folios < cur_pages) { + folio = readahead_folio(rac); + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].length = folio_size(folio); + ap->num_folios++; } - ap->num_pages = nr_pages; fuse_send_readpages(ia, rac->file); + nr_pages -= cur_pages; } } @@ -1139,8 +1171,8 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, bool short_write; int err; - for (i = 0; i < ap->num_pages; i++) - fuse_wait_on_page_writeback(inode, ap->pages[i]->index); + for (i = 0; i < ap->num_folios; i++) + fuse_wait_on_folio_writeback(inode, ap->folios[i]); fuse_write_args_fill(ia, ff, pos, count); ia->write.in.flags = fuse_write_flags(iocb); @@ -1154,24 +1186,24 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, short_write = ia->write.out.size < count; offset = ap->descs[0].offset; count = ia->write.out.size; - for (i = 0; i < ap->num_pages; i++) { - struct page *page = ap->pages[i]; + for (i = 0; i < ap->num_folios; i++) { + struct folio *folio = ap->folios[i]; if (err) { - ClearPageUptodate(page); + folio_clear_uptodate(folio); } else { - if (count >= PAGE_SIZE - offset) - count -= PAGE_SIZE - offset; + if (count >= folio_size(folio) - offset) + count -= folio_size(folio) - offset; else { if (short_write) - ClearPageUptodate(page); + folio_clear_uptodate(folio); count = 0; } offset = 0; } - if (ia->write.page_locked && (i == ap->num_pages - 1)) - unlock_page(page); - put_page(page); + if (ia->write.folio_locked && (i == ap->num_folios - 1)) + folio_unlock(folio); + folio_put(folio); } return err; @@ -1185,6 +1217,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, struct fuse_args_pages *ap = &ia->ap; struct fuse_conn *fc = get_fuse_conn(mapping->host); unsigned offset = pos & (PAGE_SIZE - 1); + unsigned int nr_pages = 0; size_t count = 0; int err; @@ -1193,7 +1226,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, do { size_t tmp; - struct page *page; + struct folio *folio; pgoff_t index = pos >> PAGE_SHIFT; size_t bytes = min_t(size_t, PAGE_SIZE - offset, iov_iter_count(ii)); @@ -1205,27 +1238,30 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, if (fault_in_iov_iter_readable(ii, bytes)) break; - err = -ENOMEM; - page = grab_cache_page_write_begin(mapping, index); - if (!page) + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); break; + } if (mapping_writably_mapped(mapping)) - flush_dcache_page(page); + flush_dcache_folio(folio); - tmp = copy_page_from_iter_atomic(page, offset, bytes, ii); - flush_dcache_page(page); + tmp = copy_folio_from_iter_atomic(folio, offset, bytes, ii); + flush_dcache_folio(folio); if (!tmp) { - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); goto again; } err = 0; - ap->pages[ap->num_pages] = page; - ap->descs[ap->num_pages].length = tmp; - ap->num_pages++; + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].length = tmp; + ap->num_folios++; + nr_pages++; count += tmp; pos += tmp; @@ -1235,18 +1271,18 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, /* If we copied full page, mark it uptodate */ if (tmp == PAGE_SIZE) - SetPageUptodate(page); + folio_mark_uptodate(folio); - if (PageUptodate(page)) { - unlock_page(page); + if (folio_test_uptodate(folio)) { + folio_unlock(folio); } else { - ia->write.page_locked = true; + ia->write.folio_locked = true; break; } if (!fc->big_writes) break; } while (iov_iter_count(ii) && count < fc->max_write && - ap->num_pages < max_pages && offset == 0); + nr_pages < max_pages && offset == 0); return count > 0 ? count : err; } @@ -1280,8 +1316,8 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), fc->max_pages); - ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs); - if (!ap->pages) { + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) { err = -ENOMEM; break; } @@ -1303,7 +1339,7 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) err = -EIO; } } - kfree(ap->pages); + kfree(ap->folios); } while (!err && iov_iter_count(ii)); fuse_write_update_attr(inode, pos, res); @@ -1430,11 +1466,7 @@ writethrough: task_io_account_write(count); - err = file_remove_privs(file); - if (err) - goto out; - - err = file_update_time(file); + err = kiocb_modified(iocb); if (err) goto out; @@ -1468,35 +1500,57 @@ static inline size_t fuse_get_frag_size(const struct iov_iter *ii, static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, size_t *nbytesp, int write, - unsigned int max_pages) + unsigned int max_pages, + bool use_pages_for_kvec_io) { + bool flush_or_invalidate = false; + unsigned int nr_pages = 0; size_t nbytes = 0; /* # bytes already packed in req */ ssize_t ret = 0; - /* Special case for kernel I/O: can copy directly into the buffer */ + /* Special case for kernel I/O: can copy directly into the buffer. + * However if the implementation of fuse_conn requires pages instead of + * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead. + */ if (iov_iter_is_kvec(ii)) { - unsigned long user_addr = fuse_get_user_addr(ii); - size_t frag_size = fuse_get_frag_size(ii, *nbytesp); + void *user_addr = (void *)fuse_get_user_addr(ii); - if (write) - ap->args.in_args[1].value = (void *) user_addr; - else - ap->args.out_args[0].value = (void *) user_addr; + if (!use_pages_for_kvec_io) { + size_t frag_size = fuse_get_frag_size(ii, *nbytesp); - iov_iter_advance(ii, frag_size); - *nbytesp = frag_size; - return 0; + if (write) + ap->args.in_args[1].value = user_addr; + else + ap->args.out_args[0].value = user_addr; + + iov_iter_advance(ii, frag_size); + *nbytesp = frag_size; + return 0; + } + + if (is_vmalloc_addr(user_addr)) { + ap->args.vmap_base = user_addr; + flush_or_invalidate = true; + } } - while (nbytes < *nbytesp && ap->num_pages < max_pages) { - unsigned npages; + /* + * Until there is support for iov_iter_extract_folios(), we have to + * manually extract pages using iov_iter_extract_pages() and then + * copy that to a folios array. + */ + struct page **pages = kzalloc(max_pages * sizeof(struct page *), + GFP_KERNEL); + if (!pages) + return -ENOMEM; + + while (nbytes < *nbytesp && nr_pages < max_pages) { + unsigned nfolios, i; size_t start; - struct page **pt_pages; - pt_pages = &ap->pages[ap->num_pages]; - ret = iov_iter_extract_pages(ii, &pt_pages, + ret = iov_iter_extract_pages(ii, &pages, *nbytesp - nbytes, - max_pages - ap->num_pages, + max_pages - nr_pages, 0, &start); if (ret < 0) break; @@ -1504,16 +1558,25 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, nbytes += ret; ret += start; - npages = DIV_ROUND_UP(ret, PAGE_SIZE); + /* Currently, all folios in FUSE are one page */ + nfolios = DIV_ROUND_UP(ret, PAGE_SIZE); - ap->descs[ap->num_pages].offset = start; - fuse_page_descs_length_init(ap->descs, ap->num_pages, npages); + ap->descs[ap->num_folios].offset = start; + fuse_folio_descs_length_init(ap->descs, ap->num_folios, nfolios); + for (i = 0; i < nfolios; i++) + ap->folios[i + ap->num_folios] = page_folio(pages[i]); - ap->num_pages += npages; - ap->descs[ap->num_pages - 1].length -= + ap->num_folios += nfolios; + ap->descs[ap->num_folios - 1].length -= (PAGE_SIZE - ret) & (PAGE_SIZE - 1); + nr_pages += nfolios; } + kfree(pages); + + if (write && flush_or_invalidate) + flush_kernel_vmap_range(ap->args.vmap_base, nbytes); + ap->args.invalidate_vmap = !write && flush_or_invalidate; ap->args.is_pinned = iov_iter_extract_will_pin(ii); ap->args.user_pages = true; if (write) @@ -1582,7 +1645,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, size_t nbytes = min(count, nmax); err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, - max_pages); + max_pages, fc->use_pages_for_kvec_io); if (err && !nbytes) break; @@ -1596,7 +1659,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, } if (!io->async || nres < 0) { - fuse_release_user_pages(&ia->ap, io->should_dirty); + fuse_release_user_pages(&ia->ap, nres, io->should_dirty); fuse_io_free(ia); } ia = NULL; @@ -1650,7 +1713,7 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) { ssize_t res; - if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { + if (!is_sync_kiocb(iocb)) { res = fuse_direct_IO(iocb, to); } else { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); @@ -1664,7 +1727,6 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); - struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); ssize_t res; bool exclusive; @@ -1672,9 +1734,11 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) res = generic_write_checks(iocb, from); if (res > 0) { task_io_account_write(res); - if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { + if (!is_sync_kiocb(iocb)) { res = fuse_direct_IO(iocb, from); } else { + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); fuse_write_update_attr(inode, iocb->ki_pos, res); @@ -1760,21 +1824,21 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) if (wpa->bucket) fuse_sync_bucket_dec(wpa->bucket); - for (i = 0; i < ap->num_pages; i++) - __free_page(ap->pages[i]); + for (i = 0; i < ap->num_folios; i++) + folio_put(ap->folios[i]); fuse_file_put(wpa->ia.ff, false); - kfree(ap->pages); + kfree(ap->folios); kfree(wpa); } -static void fuse_writepage_finish_stat(struct inode *inode, struct page *page) +static void fuse_writepage_finish_stat(struct inode *inode, struct folio *folio) { struct backing_dev_info *bdi = inode_to_bdi(inode); dec_wb_stat(&bdi->wb, WB_WRITEBACK); - dec_node_page_state(page, NR_WRITEBACK_TEMP); + node_stat_sub_folio(folio, NR_WRITEBACK_TEMP); wb_writeout_inc(&bdi->wb); } @@ -1785,8 +1849,8 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) struct fuse_inode *fi = get_fuse_inode(inode); int i; - for (i = 0; i < ap->num_pages; i++) - fuse_writepage_finish_stat(inode, ap->pages[i]); + for (i = 0; i < ap->num_folios; i++) + fuse_writepage_finish_stat(inode, ap->folios[i]); wake_up(&fi->page_waitq); } @@ -1801,7 +1865,8 @@ __acquires(fi->lock) struct fuse_inode *fi = get_fuse_inode(wpa->inode); struct fuse_write_in *inarg = &wpa->ia.write.in; struct fuse_args *args = &wpa->ia.ap.args; - __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE; + /* Currently, all folios in FUSE are one page */ + __u64 data_size = wpa->ia.ap.num_folios * PAGE_SIZE; int err; fi->writectr++; @@ -1841,7 +1906,8 @@ __acquires(fi->lock) for (aux = wpa->next; aux; aux = next) { next = aux->next; aux->next = NULL; - fuse_writepage_finish_stat(aux->inode, aux->ia.ap.pages[0]); + fuse_writepage_finish_stat(aux->inode, + aux->ia.ap.folios[0]); fuse_writepage_free(aux); } @@ -1876,11 +1942,11 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, struct fuse_writepage_args *wpa) { pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT; - pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1; + pgoff_t idx_to = idx_from + wpa->ia.ap.num_folios - 1; struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; - WARN_ON(!wpa->ia.ap.num_pages); + WARN_ON(!wpa->ia.ap.num_folios); while (*p) { struct fuse_writepage_args *curr; pgoff_t curr_index; @@ -1891,7 +1957,7 @@ static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, WARN_ON(curr->inode != wpa->inode); curr_index = curr->ia.write.in.offset >> PAGE_SHIFT; - if (idx_from >= curr_index + curr->ia.ap.num_pages) + if (idx_from >= curr_index + curr->ia.ap.num_folios) p = &(*p)->rb_right; else if (idx_to < curr_index) p = &(*p)->rb_left; @@ -2023,9 +2089,9 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) wpa = kzalloc(sizeof(*wpa), GFP_NOFS); if (wpa) { ap = &wpa->ia.ap; - ap->num_pages = 0; - ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs); - if (!ap->pages) { + ap->num_folios = 0; + ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); + if (!ap->folios) { kfree(wpa); wpa = NULL; } @@ -2049,19 +2115,19 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, } static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, - struct folio *tmp_folio, uint32_t page_index) + struct folio *tmp_folio, uint32_t folio_index) { struct inode *inode = folio->mapping->host; struct fuse_args_pages *ap = &wpa->ia.ap; folio_copy(tmp_folio, folio); - ap->pages[page_index] = &tmp_folio->page; - ap->descs[page_index].offset = 0; - ap->descs[page_index].length = PAGE_SIZE; + ap->folios[folio_index] = tmp_folio; + ap->descs[folio_index].offset = 0; + ap->descs[folio_index].length = PAGE_SIZE; inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); - inc_node_page_state(&tmp_folio->page, NR_WRITEBACK_TEMP); + node_stat_add_folio(tmp_folio, NR_WRITEBACK_TEMP); } static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, @@ -2115,7 +2181,7 @@ static int fuse_writepage_locked(struct folio *folio) goto err_writepage_args; ap = &wpa->ia.ap; - ap->num_pages = 1; + ap->num_folios = 1; folio_start_writeback(folio); fuse_writepage_args_page_fill(wpa, folio, tmp_folio, 0); @@ -2143,32 +2209,32 @@ struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; struct inode *inode; - struct page **orig_pages; - unsigned int max_pages; + struct folio **orig_folios; + unsigned int max_folios; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) { struct fuse_args_pages *ap = &data->wpa->ia.ap; struct fuse_conn *fc = get_fuse_conn(data->inode); - struct page **pages; - struct fuse_page_desc *descs; - unsigned int npages = min_t(unsigned int, - max_t(unsigned int, data->max_pages * 2, - FUSE_DEFAULT_MAX_PAGES_PER_REQ), + struct folio **folios; + struct fuse_folio_desc *descs; + unsigned int nfolios = min_t(unsigned int, + max_t(unsigned int, data->max_folios * 2, + FUSE_DEFAULT_MAX_PAGES_PER_REQ), fc->max_pages); - WARN_ON(npages <= data->max_pages); + WARN_ON(nfolios <= data->max_folios); - pages = fuse_pages_alloc(npages, GFP_NOFS, &descs); - if (!pages) + folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); + if (!folios) return false; - memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages); - memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages); - kfree(ap->pages); - ap->pages = pages; + memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios); + memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios); + kfree(ap->folios); + ap->folios = folios; ap->descs = descs; - data->max_pages = npages; + data->max_folios = nfolios; return true; } @@ -2178,7 +2244,7 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) struct fuse_writepage_args *wpa = data->wpa; struct inode *inode = data->inode; struct fuse_inode *fi = get_fuse_inode(inode); - int num_pages = wpa->ia.ap.num_pages; + int num_folios = wpa->ia.ap.num_folios; int i; spin_lock(&fi->lock); @@ -2186,8 +2252,8 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) fuse_flush_writepages(inode); spin_unlock(&fi->lock); - for (i = 0; i < num_pages; i++) - end_page_writeback(data->orig_pages[i]); + for (i = 0; i < num_folios; i++) + folio_end_writeback(data->orig_folios[i]); } /* @@ -2198,15 +2264,15 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) * swapping the new temp page with the old one. */ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, - struct page *page) + struct folio *folio) { struct fuse_inode *fi = get_fuse_inode(new_wpa->inode); struct fuse_writepage_args *tmp; struct fuse_writepage_args *old_wpa; struct fuse_args_pages *new_ap = &new_wpa->ia.ap; - WARN_ON(new_ap->num_pages != 0); - new_ap->num_pages = 1; + WARN_ON(new_ap->num_folios != 0); + new_ap->num_folios = 1; spin_lock(&fi->lock); old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa); @@ -2220,9 +2286,9 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, WARN_ON(tmp->inode != new_wpa->inode); curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT; - if (curr_index == page->index) { - WARN_ON(tmp->ia.ap.num_pages != 1); - swap(tmp->ia.ap.pages[0], new_ap->pages[0]); + if (curr_index == folio->index) { + WARN_ON(tmp->ia.ap.num_folios != 1); + swap(tmp->ia.ap.folios[0], new_ap->folios[0]); break; } } @@ -2235,18 +2301,19 @@ static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, spin_unlock(&fi->lock); if (tmp) { - fuse_writepage_finish_stat(new_wpa->inode, new_ap->pages[0]); + fuse_writepage_finish_stat(new_wpa->inode, + folio); fuse_writepage_free(new_wpa); } return false; } -static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, +static bool fuse_writepage_need_send(struct fuse_conn *fc, struct folio *folio, struct fuse_args_pages *ap, struct fuse_fill_wb_data *data) { - WARN_ON(!ap->num_pages); + WARN_ON(!ap->num_folios); /* * Being under writeback is unlikely but possible. For example direct @@ -2254,23 +2321,23 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, * the pages are faulted with get_user_pages(), and then after the read * completed. */ - if (fuse_page_is_writeback(data->inode, page->index)) + if (fuse_folio_is_writeback(data->inode, folio)) return true; /* Reached max pages */ - if (ap->num_pages == fc->max_pages) + if (ap->num_folios == fc->max_pages) return true; /* Reached max write bytes */ - if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write) + if ((ap->num_folios + 1) * PAGE_SIZE > fc->max_write) return true; /* Discontinuity */ - if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index) + if (data->orig_folios[ap->num_folios - 1]->index + 1 != folio_index(folio)) return true; /* Need to grow the pages array? If so, did the expansion fail? */ - if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data)) + if (ap->num_folios == data->max_folios && !fuse_pages_realloc(data)) return true; return false; @@ -2295,7 +2362,7 @@ static int fuse_writepages_fill(struct folio *folio, goto out_unlock; } - if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data)) { + if (wpa && fuse_writepage_need_send(fc, folio, ap, data)) { fuse_writepages_send(data); data->wpa = NULL; } @@ -2314,7 +2381,7 @@ static int fuse_writepages_fill(struct folio *folio, * This is ensured by holding the page lock in page_mkwrite() while * checking fuse_page_is_writeback(). We already hold the page lock * since clear_page_dirty_for_io() and keep it held until we add the - * request to the fi->writepages list and increment ap->num_pages. + * request to the fi->writepages list and increment ap->num_folios. * After this fuse_page_is_writeback() will indicate that the page is * under writeback, so we can release the page lock. */ @@ -2326,13 +2393,13 @@ static int fuse_writepages_fill(struct folio *folio, goto out_unlock; } fuse_file_get(wpa->ia.ff); - data->max_pages = 1; + data->max_folios = 1; ap = &wpa->ia.ap; } folio_start_writeback(folio); - fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_pages); - data->orig_pages[ap->num_pages] = &folio->page; + fuse_writepage_args_page_fill(wpa, folio, tmp_folio, ap->num_folios); + data->orig_folios[ap->num_folios] = folio; err = 0; if (data->wpa) { @@ -2341,9 +2408,9 @@ static int fuse_writepages_fill(struct folio *folio, * fuse_page_is_writeback(). */ spin_lock(&fi->lock); - ap->num_pages++; + ap->num_folios++; spin_unlock(&fi->lock); - } else if (fuse_writepage_add(wpa, &folio->page)) { + } else if (fuse_writepage_add(wpa, folio)) { data->wpa = wpa; } else { folio_end_writeback(folio); @@ -2375,21 +2442,21 @@ static int fuse_writepages(struct address_space *mapping, data.ff = NULL; err = -ENOMEM; - data.orig_pages = kcalloc(fc->max_pages, - sizeof(struct page *), - GFP_NOFS); - if (!data.orig_pages) + data.orig_folios = kcalloc(fc->max_pages, + sizeof(struct folio *), + GFP_NOFS); + if (!data.orig_folios) goto out; err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); if (data.wpa) { - WARN_ON(!data.wpa->ia.ap.num_pages); + WARN_ON(!data.wpa->ia.ap.num_folios); fuse_writepages_send(&data); } if (data.ff) fuse_file_put(data.ff, false); - kfree(data.orig_pages); + kfree(data.orig_folios); out: return err; } @@ -2429,7 +2496,7 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, folio_zero_segment(folio, 0, off); goto success; } - err = fuse_do_readpage(file, &folio->page); + err = fuse_do_readfolio(file, folio); if (err) goto cleanup; success: @@ -2518,17 +2585,17 @@ static void fuse_vma_close(struct vm_area_struct *vma) */ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) { - struct page *page = vmf->page; + struct folio *folio = page_folio(vmf->page); struct inode *inode = file_inode(vmf->vma->vm_file); file_update_time(vmf->vma->vm_file); - lock_page(page); - if (page->mapping != inode->i_mapping) { - unlock_page(page); + folio_lock(folio); + if (folio->mapping != inode->i_mapping) { + folio_unlock(folio); return VM_FAULT_NOPAGE; } - fuse_wait_on_page_writeback(inode, page->index); + fuse_wait_on_folio_writeback(inode, folio); return VM_FAULT_LOCKED; } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index e6cc3d552b13..74744c6f2860 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -35,9 +35,6 @@ /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 -/** Maximum of max_pages received in init_out */ -#define FUSE_MAX_MAX_PAGES 256 - /** Bias for fi->writectr, meaning new writepages must not be sent */ #define FUSE_NOWRITE INT_MIN @@ -47,6 +44,9 @@ /** Number of dentries for each connection in the control filesystem */ #define FUSE_CTL_NUM_DENTRIES 5 +/** Maximum of max_pages received in init_out */ +extern unsigned int fuse_max_pages_limit; + /** List of active connections */ extern struct list_head fuse_conn_list; @@ -285,8 +285,8 @@ struct fuse_arg { void *value; }; -/** FUSE page descriptor */ -struct fuse_page_desc { +/** FUSE folio descriptor */ +struct fuse_folio_desc { unsigned int length; unsigned int offset; }; @@ -309,16 +309,19 @@ struct fuse_args { bool may_block:1; bool is_ext:1; bool is_pinned:1; + bool invalidate_vmap:1; struct fuse_in_arg in_args[3]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); + /* Used for kvec iter backed by vmalloc address */ + void *vmap_base; }; struct fuse_args_pages { struct fuse_args args; - struct page **pages; - struct fuse_page_desc *descs; - unsigned int num_pages; + struct folio **folios; + struct fuse_folio_desc *descs; + unsigned int num_folios; }; struct fuse_release_args { @@ -857,6 +860,9 @@ struct fuse_conn { /** Passthrough support for read/write IO */ unsigned int passthrough:1; + /* Use pages instead of pointer for kernel I/O */ + unsigned int use_pages_for_kvec_io:1; + /** Maximum stack depth for passthrough backing files */ int max_stack_depth; @@ -884,6 +890,9 @@ struct fuse_conn { /** Version counter for attribute changes */ atomic64_t attr_version; + /** Version counter for evict inode */ + atomic64_t evict_ctr; + /** Called on final put */ void (*release)(struct fuse_conn *); @@ -978,6 +987,11 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) return atomic64_read(&fc->attr_version); } +static inline u64 fuse_get_evict_ctr(struct fuse_conn *fc) +{ + return atomic64_read(&fc->evict_ctr); +} + static inline bool fuse_stale_inode(const struct inode *inode, int generation, struct fuse_attr *attr) { @@ -995,25 +1009,25 @@ static inline bool fuse_is_bad(struct inode *inode) return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state)); } -static inline struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags, - struct fuse_page_desc **desc) +static inline struct folio **fuse_folios_alloc(unsigned int nfolios, gfp_t flags, + struct fuse_folio_desc **desc) { - struct page **pages; + struct folio **folios; - pages = kzalloc(npages * (sizeof(struct page *) + - sizeof(struct fuse_page_desc)), flags); - *desc = (void *) (pages + npages); + folios = kzalloc(nfolios * (sizeof(struct folio *) + + sizeof(struct fuse_folio_desc)), flags); + *desc = (void *) (folios + nfolios); - return pages; + return folios; } -static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs, - unsigned int index, - unsigned int nr_pages) +static inline void fuse_folio_descs_length_init(struct fuse_folio_desc *descs, + unsigned int index, + unsigned int nr_folios) { int i; - for (i = index; i < index + nr_pages; i++) + for (i = index; i < index + nr_folios; i++) descs[i].length = PAGE_SIZE - descs[i].offset; } @@ -1037,7 +1051,8 @@ extern const struct dentry_operations fuse_root_dentry_operations; */ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, int generation, struct fuse_attr *attr, - u64 attr_valid, u64 attr_version); + u64 attr_valid, u64 attr_version, + u64 evict_ctr); int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, struct fuse_entry_out *outarg, struct inode **inode); @@ -1062,7 +1077,7 @@ struct fuse_io_args { struct { struct fuse_write_in in; struct fuse_write_out out; - bool page_locked; + bool folio_locked; } write; }; struct fuse_args_pages ap; @@ -1127,7 +1142,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, - u64 attr_valid, u32 cache_mask); + u64 attr_valid, u32 cache_mask, + u64 evict_ctr); u32 fuse_get_cache_mask(struct inode *inode); @@ -1480,4 +1496,12 @@ ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, size_t len, unsigned int flags); ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); +#ifdef CONFIG_SYSCTL +extern int fuse_sysctl_register(void); +extern void fuse_sysctl_unregister(void); +#else +#define fuse_sysctl_register() (0) +#define fuse_sysctl_unregister() do { } while (0) +#endif /* CONFIG_SYSCTL */ + #endif /* _FS_FUSE_I_H */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index fd3321e29a3e..3ce4f4e81d09 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -35,6 +35,8 @@ DEFINE_MUTEX(fuse_mutex); static int set_global_limit(const char *val, const struct kernel_param *kp); +unsigned int fuse_max_pages_limit = 256; + unsigned max_user_bgreq; module_param_call(max_user_bgreq, set_global_limit, param_get_uint, &max_user_bgreq, 0644); @@ -173,6 +175,14 @@ static void fuse_evict_inode(struct inode *inode) fuse_cleanup_submount_lookup(fc, fi->submount_lookup); fi->submount_lookup = NULL; } + /* + * Evict of non-deleted inode may race with outstanding + * LOOKUP/READDIRPLUS requests and result in inconsistency when + * the request finishes. Deal with that here by bumping a + * counter that can be compared to the starting value. + */ + if (inode->i_nlink > 0) + atomic64_inc(&fc->evict_ctr); } if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) { WARN_ON(fi->iocachectr != 0); @@ -206,17 +216,30 @@ static ino_t fuse_squash_ino(u64 ino64) void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, - u64 attr_valid, u32 cache_mask) + u64 attr_valid, u32 cache_mask, + u64 evict_ctr) { struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); lockdep_assert_held(&fi->lock); + /* + * Clear basic stats from invalid mask. + * + * Don't do this if this is coming from a fuse_iget() call and there + * might have been a racing evict which would've invalidated the result + * if the attr_version would've been preserved. + * + * !evict_ctr -> this is create + * fi->attr_version != 0 -> this is not a new inode + * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request + */ + if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) + set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); + fi->attr_version = atomic64_inc_return(&fc->attr_version); fi->i_time = attr_valid; - /* Clear basic stats from invalid mask */ - set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); inode->i_ino = fuse_squash_ino(attr->ino); inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); @@ -295,9 +318,9 @@ u32 fuse_get_cache_mask(struct inode *inode) return STATX_MTIME | STATX_CTIME | STATX_SIZE; } -void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, - struct fuse_statx *sx, - u64 attr_valid, u64 attr_version) +static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, u64 attr_valid, + u64 attr_version, u64 evict_ctr) { struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); @@ -331,7 +354,8 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, } old_mtime = inode_get_mtime(inode); - fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask); + fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, + evict_ctr); oldsize = inode->i_size; /* @@ -372,6 +396,13 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, fuse_dax_dontcache(inode, attr->flags); } +void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, u64 attr_valid, + u64 attr_version) +{ + fuse_change_attributes_i(inode, attr, sx, attr_valid, attr_version, 0); +} + static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl, u64 nodeid) { @@ -426,7 +457,8 @@ static int fuse_inode_set(struct inode *inode, void *_nodeidp) struct inode *fuse_iget(struct super_block *sb, u64 nodeid, int generation, struct fuse_attr *attr, - u64 attr_valid, u64 attr_version) + u64 attr_valid, u64 attr_version, + u64 evict_ctr) { struct inode *inode; struct fuse_inode *fi; @@ -487,8 +519,8 @@ retry: fi->nlookup++; spin_unlock(&fi->lock); done: - fuse_change_attributes(inode, attr, NULL, attr_valid, attr_version); - + fuse_change_attributes_i(inode, attr, NULL, attr_valid, attr_version, + evict_ctr); return inode; } @@ -940,11 +972,12 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->initialized = 0; fc->connected = 1; atomic64_set(&fc->attr_version, 1); + atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); fc->pid_ns = get_pid_ns(task_active_pid_ns(current)); fc->user_ns = get_user_ns(user_ns); fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; - fc->max_pages_limit = FUSE_MAX_MAX_PAGES; + fc->max_pages_limit = fuse_max_pages_limit; if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_init(fc); @@ -1001,7 +1034,7 @@ static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned mode) attr.mode = mode; attr.ino = FUSE_ROOT_ID; attr.nlink = 1; - return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0); + return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0, 0); } struct fuse_inode_handle { @@ -1610,7 +1643,8 @@ static int fuse_fill_super_submount(struct super_block *sb, return -ENOMEM; fuse_fill_attr_from_inode(&root_attr, parent_fi); - root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0); + root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, + fuse_get_evict_ctr(fm->fc)); /* * This inode is just a duplicate, so it is not looked up and * its nlookup should not be incremented. fuse_iget() does @@ -2063,8 +2097,14 @@ static int __init fuse_fs_init(void) if (err) goto out3; + err = fuse_sysctl_register(); + if (err) + goto out4; + return 0; + out4: + unregister_filesystem(&fuse_fs_type); out3: unregister_fuseblk(); out2: @@ -2075,6 +2115,7 @@ static int __init fuse_fs_init(void) static void fuse_fs_cleanup(void) { + fuse_sysctl_unregister(); unregister_filesystem(&fuse_fs_type); unregister_fuseblk(); diff --git a/fs/fuse/ioctl.c b/fs/fuse/ioctl.c index 572ce8a82ceb..2d9abf48828f 100644 --- a/fs/fuse/ioctl.c +++ b/fs/fuse/ioctl.c @@ -10,6 +10,8 @@ #include <linux/fileattr.h> #include <linux/fsverity.h> +#define FUSE_VERITY_ENABLE_ARG_MAX_PAGES 256 + static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args, struct fuse_ioctl_out *outarg) { @@ -140,7 +142,7 @@ static int fuse_setup_enable_verity(unsigned long arg, struct iovec *iov, { struct fsverity_enable_arg enable; struct fsverity_enable_arg __user *uarg = (void __user *)arg; - const __u32 max_buffer_len = FUSE_MAX_MAX_PAGES * PAGE_SIZE; + const __u32 max_buffer_len = FUSE_VERITY_ENABLE_ARG_MAX_PAGES * PAGE_SIZE; if (copy_from_user(&enable, uarg, sizeof(enable))) return -EFAULT; @@ -249,12 +251,12 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); err = -ENOMEM; - ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); + ap.folios = fuse_folios_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); - if (!ap.pages || !iov_page) + if (!ap.folios || !iov_page) goto out; - fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages); + fuse_folio_descs_length_init(ap.descs, 0, fm->fc->max_pages); /* * If restricted, initialize IO parameters as encoded in @cmd. @@ -304,14 +306,13 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, err = -ENOMEM; if (max_pages > fm->fc->max_pages) goto out; - while (ap.num_pages < max_pages) { - ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); - if (!ap.pages[ap.num_pages]) + while (ap.num_folios < max_pages) { + ap.folios[ap.num_folios] = folio_alloc(GFP_KERNEL | __GFP_HIGHMEM, 0); + if (!ap.folios[ap.num_folios]) goto out; - ap.num_pages++; + ap.num_folios++; } - /* okay, let's send it to the client */ ap.args.opcode = FUSE_IOCTL; ap.args.nodeid = ff->nodeid; @@ -325,8 +326,8 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, err = -EFAULT; iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size); - for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) { - c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii); + for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { + c = copy_folio_from_iter(ap.folios[i], 0, PAGE_SIZE, &ii); if (c != PAGE_SIZE && iov_iter_count(&ii)) goto out; } @@ -364,7 +365,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) goto out; - vaddr = kmap_local_page(ap.pages[0]); + vaddr = kmap_local_folio(ap.folios[0], 0); err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr, transferred, in_iovs + out_iovs, (flags & FUSE_IOCTL_COMPAT) != 0); @@ -392,17 +393,17 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, err = -EFAULT; iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred); - for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) { - c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii); + for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { + c = copy_folio_to_iter(ap.folios[i], 0, PAGE_SIZE, &ii); if (c != PAGE_SIZE && iov_iter_count(&ii)) goto out; } err = 0; out: free_page((unsigned long) iov_page); - while (ap.num_pages) - __free_page(ap.pages[--ap.num_pages]); - kfree(ap.pages); + while (ap.num_folios) + folio_put(ap.folios[--ap.num_folios]); + kfree(ap.folios); return err ? err : outarg.result; } diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0377b6dc24c8..17ce9636a2b1 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -149,7 +149,7 @@ static int parse_dirfile(char *buf, size_t nbytes, struct file *file, static int fuse_direntplus_link(struct file *file, struct fuse_direntplus *direntplus, - u64 attr_version) + u64 attr_version, u64 evict_ctr) { struct fuse_entry_out *o = &direntplus->entry_out; struct fuse_dirent *dirent = &direntplus->dirent; @@ -233,7 +233,7 @@ retry: } else { inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, &o->attr, ATTR_TIMEOUT(o), - attr_version); + attr_version, evict_ctr); if (!inode) inode = ERR_PTR(-ENOMEM); @@ -284,7 +284,8 @@ static void fuse_force_forget(struct file *file, u64 nodeid) } static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, - struct dir_context *ctx, u64 attr_version) + struct dir_context *ctx, u64 attr_version, + u64 evict_ctr) { struct fuse_direntplus *direntplus; struct fuse_dirent *dirent; @@ -319,7 +320,7 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, buf += reclen; nbytes -= reclen; - ret = fuse_direntplus_link(file, direntplus, attr_version); + ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr); if (ret) fuse_force_forget(file, direntplus->entry_out.nodeid); } @@ -331,26 +332,27 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus; ssize_t res; - struct page *page; + struct folio *folio; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_io_args ia = {}; struct fuse_args_pages *ap = &ia.ap; - struct fuse_page_desc desc = { .length = PAGE_SIZE }; - u64 attr_version = 0; + struct fuse_folio_desc desc = { .length = PAGE_SIZE }; + u64 attr_version = 0, evict_ctr = 0; bool locked; - page = alloc_page(GFP_KERNEL); - if (!page) + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) return -ENOMEM; plus = fuse_use_readdirplus(inode, ctx); ap->args.out_pages = true; - ap->num_pages = 1; - ap->pages = &page; + ap->num_folios = 1; + ap->folios = &folio; ap->descs = &desc; if (plus) { attr_version = fuse_get_attr_version(fm->fc); + evict_ctr = fuse_get_evict_ctr(fm->fc); fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE, FUSE_READDIRPLUS); } else { @@ -367,15 +369,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) if (ff->open_flags & FOPEN_CACHE_DIR) fuse_readdir_cache_end(file, ctx->pos); } else if (plus) { - res = parse_dirplusfile(page_address(page), res, - file, ctx, attr_version); + res = parse_dirplusfile(folio_address(folio), res, + file, ctx, attr_version, + evict_ctr); } else { - res = parse_dirfile(page_address(page), res, file, + res = parse_dirfile(folio_address(folio), res, file, ctx); } } - __free_page(page); + folio_put(folio); fuse_invalidate_atime(inode); return res; } diff --git a/fs/fuse/sysctl.c b/fs/fuse/sysctl.c new file mode 100644 index 000000000000..b272bb333005 --- /dev/null +++ b/fs/fuse/sysctl.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/fuse/fuse_sysctl.c + * + * Sysctl interface to fuse parameters + */ +#include <linux/sysctl.h> + +#include "fuse_i.h" + +static struct ctl_table_header *fuse_table_header; + +/* Bound by fuse_init_out max_pages, which is a u16 */ +static unsigned int sysctl_fuse_max_pages_limit = 65535; + +static struct ctl_table fuse_sysctl_table[] = { + { + .procname = "max_pages_limit", + .data = &fuse_max_pages_limit, + .maxlen = sizeof(fuse_max_pages_limit), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &sysctl_fuse_max_pages_limit, + }, +}; + +int fuse_sysctl_register(void) +{ + fuse_table_header = register_sysctl("fs/fuse", fuse_sysctl_table); + if (!fuse_table_header) + return -ENOMEM; + return 0; +} + +void fuse_sysctl_unregister(void) +{ + unregister_sysctl_table(fuse_table_header); + fuse_table_header = NULL; +} diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 6404a189e989..d88d3fc5306a 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -97,7 +97,8 @@ struct virtio_fs_req_work { }; static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, - struct fuse_req *req, bool in_flight); + struct fuse_req *req, bool in_flight, + gfp_t gfp); static const struct constant_table dax_param_enums[] = { {"always", FUSE_DAX_ALWAYS }, @@ -575,6 +576,8 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) /* Dispatch pending requests */ while (1) { + unsigned int flags; + spin_lock(&fsvq->lock); req = list_first_entry_or_null(&fsvq->queued_reqs, struct fuse_req, list); @@ -585,7 +588,9 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) list_del_init(&req->list); spin_unlock(&fsvq->lock); - ret = virtio_fs_enqueue_req(fsvq, req, true); + flags = memalloc_nofs_save(); + ret = virtio_fs_enqueue_req(fsvq, req, true, GFP_KERNEL); + memalloc_nofs_restore(flags); if (ret < 0) { if (ret == -ENOSPC) { spin_lock(&fsvq->lock); @@ -686,7 +691,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) } /* Allocate and copy args into req->argbuf */ -static int copy_args_to_argbuf(struct fuse_req *req) +static int copy_args_to_argbuf(struct fuse_req *req, gfp_t gfp) { struct fuse_args *args = req->args; unsigned int offset = 0; @@ -700,7 +705,7 @@ static int copy_args_to_argbuf(struct fuse_req *req) len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + fuse_len_args(num_out, args->out_args); - req->argbuf = kmalloc(len, GFP_ATOMIC); + req->argbuf = kmalloc(len, gfp); if (!req->argbuf) return -ENOMEM; @@ -760,7 +765,7 @@ static void virtio_fs_request_complete(struct fuse_req *req, struct fuse_args *args; struct fuse_args_pages *ap; unsigned int len, i, thislen; - struct page *page; + struct folio *folio; /* * TODO verify that server properly follows FUSE protocol @@ -772,12 +777,12 @@ static void virtio_fs_request_complete(struct fuse_req *req, if (args->out_pages && args->page_zeroing) { len = args->out_args[args->out_numargs - 1].size; ap = container_of(args, typeof(*ap), args); - for (i = 0; i < ap->num_pages; i++) { + for (i = 0; i < ap->num_folios; i++) { thislen = ap->descs[i].length; if (len < thislen) { WARN_ON(ap->descs[i].offset); - page = ap->pages[i]; - zero_user_segment(page, len, thislen); + folio = ap->folios[i]; + folio_zero_segment(folio, len, thislen); len = 0; } else { len -= thislen; @@ -1267,15 +1272,15 @@ static void virtio_fs_send_interrupt(struct fuse_iqueue *fiq, struct fuse_req *r } /* Count number of scatter-gather elements required */ -static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, - unsigned int num_pages, - unsigned int total_len) +static unsigned int sg_count_fuse_folios(struct fuse_folio_desc *folio_descs, + unsigned int num_folios, + unsigned int total_len) { unsigned int i; unsigned int this_len; - for (i = 0; i < num_pages && total_len; i++) { - this_len = min(page_descs[i].length, total_len); + for (i = 0; i < num_folios && total_len; i++) { + this_len = min(folio_descs[i].length, total_len); total_len -= this_len; } @@ -1294,8 +1299,8 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) if (args->in_pages) { size = args->in_args[args->in_numargs - 1].size; - total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, - size); + total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, + size); } if (!test_bit(FR_ISREPLY, &req->flags)) @@ -1308,27 +1313,27 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) if (args->out_pages) { size = args->out_args[args->out_numargs - 1].size; - total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, - size); + total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, + size); } return total_sgs; } -/* Add pages to scatter-gather list and return number of elements used */ -static unsigned int sg_init_fuse_pages(struct scatterlist *sg, - struct page **pages, - struct fuse_page_desc *page_descs, - unsigned int num_pages, - unsigned int total_len) +/* Add folios to scatter-gather list and return number of elements used */ +static unsigned int sg_init_fuse_folios(struct scatterlist *sg, + struct folio **folios, + struct fuse_folio_desc *folio_descs, + unsigned int num_folios, + unsigned int total_len) { unsigned int i; unsigned int this_len; - for (i = 0; i < num_pages && total_len; i++) { + for (i = 0; i < num_folios && total_len; i++) { sg_init_table(&sg[i], 1); - this_len = min(page_descs[i].length, total_len); - sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); + this_len = min(folio_descs[i].length, total_len); + sg_set_folio(&sg[i], folios[i], this_len, folio_descs[i].offset); total_len -= this_len; } @@ -1353,10 +1358,10 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, sg_init_one(&sg[total_sgs++], argbuf, len); if (argpages) - total_sgs += sg_init_fuse_pages(&sg[total_sgs], - ap->pages, ap->descs, - ap->num_pages, - args[numargs - 1].size); + total_sgs += sg_init_fuse_folios(&sg[total_sgs], + ap->folios, ap->descs, + ap->num_folios, + args[numargs - 1].size); if (len_used) *len_used = len; @@ -1366,7 +1371,8 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg, /* Add a request to a virtqueue and kick the device */ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, - struct fuse_req *req, bool in_flight) + struct fuse_req *req, bool in_flight, + gfp_t gfp) { /* requests need at least 4 elements */ struct scatterlist *stack_sgs[6]; @@ -1387,8 +1393,8 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, /* Does the sglist fit on the stack? */ total_sgs = sg_count_fuse_req(req); if (total_sgs > ARRAY_SIZE(stack_sgs)) { - sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); - sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); + sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), gfp); + sg = kmalloc_array(total_sgs, sizeof(sg[0]), gfp); if (!sgs || !sg) { ret = -ENOMEM; goto out; @@ -1396,7 +1402,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, } /* Use a bounce buffer since stack args cannot be mapped */ - ret = copy_args_to_argbuf(req); + ret = copy_args_to_argbuf(req, gfp); if (ret < 0) goto out; @@ -1490,7 +1496,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) queue_id); fsvq = &fs->vqs[queue_id]; - ret = virtio_fs_enqueue_req(fsvq, req, false); + ret = virtio_fs_enqueue_req(fsvq, req, false, GFP_ATOMIC); if (ret < 0) { if (ret == -ENOSPC) { /* @@ -1691,6 +1697,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc) fc->delete_stale = true; fc->auto_submounts = true; fc->sync_fs = true; + fc->use_pages_for_kvec_io = true; /* Tell FUSE to split requests that exceed the virtqueue's size */ fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 4701c4aafbf4..8c4c1f871a88 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -35,6 +35,7 @@ #include <linux/rhashtable.h> #include <linux/pid_namespace.h> #include <linux/file.h> +#include <linux/random.h> #include "gfs2.h" #include "incore.h" @@ -562,11 +563,11 @@ static void state_change(struct gfs2_glock *gl, unsigned int new_state) gl->gl_tchange = jiffies; } -static void gfs2_set_demote(struct gfs2_glock *gl) +static void gfs2_set_demote(int nr, struct gfs2_glock *gl) { struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; - set_bit(GLF_DEMOTE, &gl->gl_flags); + set_bit(nr, &gl->gl_flags); smp_mb(); wake_up(&sdp->sd_async_glock_wait); } @@ -958,20 +959,22 @@ static void gfs2_glock_poke(struct gfs2_glock *gl) gfs2_holder_uninit(&gh); } -static bool gfs2_try_evict(struct gfs2_glock *gl) +static void gfs2_try_evict(struct gfs2_glock *gl) { struct gfs2_inode *ip; - bool evicted = false; /* * If there is contention on the iopen glock and we have an inode, try - * to grab and release the inode so that it can be evicted. This will - * allow the remote node to go ahead and delete the inode without us - * having to do it, which will avoid rgrp glock thrashing. + * to grab and release the inode so that it can be evicted. The + * GIF_DEFER_DELETE flag indicates to gfs2_evict_inode() that the inode + * should not be deleted locally. This will allow the remote node to + * go ahead and delete the inode without us having to do it, which will + * avoid rgrp glock thrashing. * * The remote node is likely still holding the corresponding inode * glock, so it will run before we get to verify that the delete has - * happened below. + * happened below. (Verification is triggered by the call to + * gfs2_queue_verify_delete() in gfs2_evict_inode().) */ spin_lock(&gl->gl_lockref.lock); ip = gl->gl_object; @@ -979,8 +982,14 @@ static bool gfs2_try_evict(struct gfs2_glock *gl) ip = NULL; spin_unlock(&gl->gl_lockref.lock); if (ip) { - gl->gl_no_formal_ino = ip->i_no_formal_ino; - set_bit(GIF_DEFERRED_DELETE, &ip->i_flags); + wait_on_inode(&ip->i_inode); + if (is_bad_inode(&ip->i_inode)) { + iput(&ip->i_inode); + ip = NULL; + } + } + if (ip) { + set_bit(GIF_DEFER_DELETE, &ip->i_flags); d_prune_aliases(&ip->i_inode); iput(&ip->i_inode); @@ -988,7 +997,7 @@ static bool gfs2_try_evict(struct gfs2_glock *gl) spin_lock(&gl->gl_lockref.lock); ip = gl->gl_object; if (ip) { - clear_bit(GIF_DEFERRED_DELETE, &ip->i_flags); + clear_bit(GIF_DEFER_DELETE, &ip->i_flags); if (!igrab(&ip->i_inode)) ip = NULL; } @@ -997,9 +1006,7 @@ static bool gfs2_try_evict(struct gfs2_glock *gl) gfs2_glock_poke(ip->i_gl); iput(&ip->i_inode); } - evicted = !ip; } - return evicted; } bool gfs2_queue_try_to_evict(struct gfs2_glock *gl) @@ -1008,18 +1015,18 @@ bool gfs2_queue_try_to_evict(struct gfs2_glock *gl) if (test_and_set_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) return false; - return queue_delayed_work(sdp->sd_delete_wq, - &gl->gl_delete, 0); + return !mod_delayed_work(sdp->sd_delete_wq, &gl->gl_delete, 0); } -static bool gfs2_queue_verify_evict(struct gfs2_glock *gl) +bool gfs2_queue_verify_delete(struct gfs2_glock *gl, bool later) { struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; + unsigned long delay; - if (test_and_set_bit(GLF_VERIFY_EVICT, &gl->gl_flags)) + if (test_and_set_bit(GLF_VERIFY_DELETE, &gl->gl_flags)) return false; - return queue_delayed_work(sdp->sd_delete_wq, - &gl->gl_delete, 5 * HZ); + delay = later ? HZ + get_random_long() % (HZ * 9) : 0; + return queue_delayed_work(sdp->sd_delete_wq, &gl->gl_delete, delay); } static void delete_work_func(struct work_struct *work) @@ -1027,43 +1034,21 @@ static void delete_work_func(struct work_struct *work) struct delayed_work *dwork = to_delayed_work(work); struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete); struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; - struct inode *inode; - u64 no_addr = gl->gl_name.ln_number; + bool verify_delete = test_and_clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags); - if (test_and_clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) { - /* - * If we can evict the inode, give the remote node trying to - * delete the inode some time before verifying that the delete - * has happened. Otherwise, if we cause contention on the inode glock - * immediately, the remote node will think that we still have - * the inode in use, and so it will give up waiting. - * - * If we can't evict the inode, signal to the remote node that - * the inode is still in use. We'll later try to delete the - * inode locally in gfs2_evict_inode. - * - * FIXME: We only need to verify that the remote node has - * deleted the inode because nodes before this remote delete - * rework won't cooperate. At a later time, when we no longer - * care about compatibility with such nodes, we can skip this - * step entirely. - */ - if (gfs2_try_evict(gl)) { - if (test_bit(SDF_KILL, &sdp->sd_flags)) - goto out; - if (gfs2_queue_verify_evict(gl)) - return; - } - goto out; - } + if (test_and_clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) + gfs2_try_evict(gl); + + if (verify_delete) { + u64 no_addr = gl->gl_name.ln_number; + struct inode *inode; - if (test_and_clear_bit(GLF_VERIFY_EVICT, &gl->gl_flags)) { inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino, GFS2_BLKST_UNLINKED); if (IS_ERR(inode)) { if (PTR_ERR(inode) == -EAGAIN && !test_bit(SDF_KILL, &sdp->sd_flags) && - gfs2_queue_verify_evict(gl)) + gfs2_queue_verify_delete(gl, true)) return; } else { d_prune_aliases(inode); @@ -1071,7 +1056,6 @@ static void delete_work_func(struct work_struct *work) } } -out: gfs2_glock_put(gl); } @@ -1100,7 +1084,7 @@ static void glock_work_func(struct work_struct *work) if (!delay) { clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags); - gfs2_set_demote(gl); + gfs2_set_demote(GLF_DEMOTE, gl); } } run_queue(gl, 0); @@ -1442,10 +1426,7 @@ out: static void request_demote(struct gfs2_glock *gl, unsigned int state, unsigned long delay, bool remote) { - if (delay) - set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags); - else - gfs2_set_demote(gl); + gfs2_set_demote(delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE, gl); if (gl->gl_demote_state == LM_ST_EXCLUSIVE) { gl->gl_demote_state = state; gl->gl_demote_time = jiffies; @@ -1635,12 +1616,6 @@ int gfs2_glock_poll(struct gfs2_holder *gh) return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1; } -static inline bool needs_demote(struct gfs2_glock *gl) -{ - return (test_bit(GLF_DEMOTE, &gl->gl_flags) || - test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags)); -} - static void __gfs2_glock_dq(struct gfs2_holder *gh) { struct gfs2_glock *gl = gh->gh_gl; @@ -1649,8 +1624,8 @@ static void __gfs2_glock_dq(struct gfs2_holder *gh) /* * This holder should not be cached, so mark it for demote. - * Note: this should be done before the check for needs_demote - * below. + * Note: this should be done before the glock_needs_demote + * check below. */ if (gh->gh_flags & GL_NOCACHE) request_demote(gl, LM_ST_UNLOCKED, 0, false); @@ -1663,7 +1638,7 @@ static void __gfs2_glock_dq(struct gfs2_holder *gh) * If there hasn't been a demote request we are done. * (Let the remaining holders, if any, keep holding it.) */ - if (!needs_demote(gl)) { + if (!glock_needs_demote(gl)) { if (list_empty(&gl->gl_holders)) fast_path = 1; } @@ -2117,7 +2092,7 @@ static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp) void gfs2_cancel_delete_work(struct gfs2_glock *gl) { clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags); - clear_bit(GLF_VERIFY_EVICT, &gl->gl_flags); + clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags); if (cancel_delayed_work(&gl->gl_delete)) gfs2_glock_put(gl); } @@ -2370,7 +2345,7 @@ static const char *gflags2str(char *buf, const struct gfs2_glock *gl) *p++ = 'N'; if (test_bit(GLF_TRY_TO_EVICT, gflags)) *p++ = 'e'; - if (test_bit(GLF_VERIFY_EVICT, gflags)) + if (test_bit(GLF_VERIFY_DELETE, gflags)) *p++ = 'E'; *p = 0; return buf; diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index adf0091cc98f..c171f745650f 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -245,6 +245,7 @@ static inline int gfs2_glock_nq_init(struct gfs2_glock *gl, void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state); void gfs2_glock_complete(struct gfs2_glock *gl, int ret); bool gfs2_queue_try_to_evict(struct gfs2_glock *gl); +bool gfs2_queue_verify_delete(struct gfs2_glock *gl, bool later); void gfs2_cancel_delete_work(struct gfs2_glock *gl); void gfs2_flush_delete_work(struct gfs2_sbd *sdp); void gfs2_gl_hash_clear(struct gfs2_sbd *sdp); @@ -284,4 +285,10 @@ static inline bool gfs2_holder_queued(struct gfs2_holder *gh) void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation); bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation); +static inline bool glock_needs_demote(struct gfs2_glock *gl) +{ + return (test_bit(GLF_DEMOTE, &gl->gl_flags) || + test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags)); +} + #endif /* __GLOCK_DOT_H__ */ diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index 95d8081681dc..eb4714f299ef 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -470,7 +470,7 @@ static int gfs2_dinode_in(struct gfs2_inode *ip, const void *buf) * Returns: errno */ -int gfs2_inode_refresh(struct gfs2_inode *ip) +static int gfs2_inode_refresh(struct gfs2_inode *ip) { struct buffer_head *dibh; int error; @@ -494,11 +494,18 @@ int gfs2_inode_refresh(struct gfs2_inode *ip) static int inode_go_instantiate(struct gfs2_glock *gl) { struct gfs2_inode *ip = gl->gl_object; + struct gfs2_glock *io_gl; + int error; if (!ip) /* no inode to populate - read it in later */ return 0; - return gfs2_inode_refresh(ip); + error = gfs2_inode_refresh(ip); + if (error) + return error; + io_gl = ip->i_iopen_gh.gh_gl; + io_gl->gl_no_formal_ino = ip->i_no_formal_ino; + return 0; } static int inode_go_held(struct gfs2_holder *gh) diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index aa4ef67a34e0..4e19cce3d906 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -329,7 +329,7 @@ enum { GLF_BLOCKING = 15, GLF_UNLOCKED = 16, /* Wait for glock to be unlocked */ GLF_TRY_TO_EVICT = 17, /* iopen glocks only */ - GLF_VERIFY_EVICT = 18, /* iopen glocks only */ + GLF_VERIFY_DELETE = 18, /* iopen glocks only */ }; struct gfs2_glock { @@ -376,7 +376,7 @@ enum { GIF_SW_PAGED = 3, GIF_FREE_VFS_INODE = 5, GIF_GLOP_PENDING = 6, - GIF_DEFERRED_DELETE = 7, + GIF_DEFER_DELETE = 7, }; struct gfs2_inode { diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 1b95db2c3aac..6fbbaaad1cd0 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -750,6 +750,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, if (error) goto fail_free_inode; gfs2_cancel_delete_work(io_gl); + io_gl->gl_no_formal_ino = ip->i_no_formal_ino; retry: error = insert_inode_locked4(inode, ip->i_no_addr, iget_test, &ip->i_no_addr); diff --git a/fs/gfs2/inode.h b/fs/gfs2/inode.h index fd15d1c6b6fb..9e5e1622d50a 100644 --- a/fs/gfs2/inode.h +++ b/fs/gfs2/inode.h @@ -93,8 +93,6 @@ struct inode *gfs2_lookup_by_inum(struct gfs2_sbd *sdp, u64 no_addr, u64 no_formal_ino, unsigned int blktype); -int gfs2_inode_refresh(struct gfs2_inode *ip); - struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name, int is_root); int gfs2_permission(struct mnt_idmap *idmap, diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c index fa5134df985f..58aeeae7ed8c 100644 --- a/fs/gfs2/lock_dlm.c +++ b/fs/gfs2/lock_dlm.c @@ -224,8 +224,21 @@ static int make_mode(struct gfs2_sbd *sdp, const unsigned int lmstate) return -1; } +/* Taken from fs/dlm/lock.c. */ + +static bool middle_conversion(int cur, int req) +{ + return (cur == DLM_LOCK_PR && req == DLM_LOCK_CW) || + (cur == DLM_LOCK_CW && req == DLM_LOCK_PR); +} + +static bool down_conversion(int cur, int req) +{ + return !middle_conversion(cur, req) && req < cur; +} + static u32 make_flags(struct gfs2_glock *gl, const unsigned int gfs_flags, - const int req) + const int cur, const int req) { u32 lkf = 0; @@ -251,7 +264,14 @@ static u32 make_flags(struct gfs2_glock *gl, const unsigned int gfs_flags, if (!test_bit(GLF_INITIAL, &gl->gl_flags)) { lkf |= DLM_LKF_CONVERT; - if (test_bit(GLF_BLOCKING, &gl->gl_flags)) + + /* + * The DLM_LKF_QUECVT flag needs to be set for "first come, + * first served" semantics, but it must only be set for + * "upward" lock conversions or else DLM will reject the + * request as invalid. + */ + if (!down_conversion(cur, req)) lkf |= DLM_LKF_QUECVT; } @@ -271,13 +291,14 @@ static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state, unsigned int flags) { struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct; - int req; + int cur, req; u32 lkf; char strname[GDLM_STRNAME_BYTES] = ""; int error; + cur = make_mode(gl->gl_name.ln_sbd, gl->gl_state); req = make_mode(gl->gl_name.ln_sbd, req_state); - lkf = make_flags(gl, flags, req); + lkf = make_flags(gl, flags, cur, req); gfs2_glstats_inc(gl, GFS2_LKS_DCOUNT); gfs2_sbstats_inc(gl, GFS2_LKS_DCOUNT); if (test_bit(GLF_INITIAL, &gl->gl_flags)) { diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 29c772816765..b14e54b38ee8 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -1879,7 +1879,7 @@ static void try_rgrp_unlink(struct gfs2_rgrpd *rgd, u64 *last_unlinked, u64 skip */ ip = gl->gl_object; - if (ip || !gfs2_queue_try_to_evict(gl)) + if (ip || !gfs2_queue_verify_delete(gl, false)) gfs2_glock_put(gl); else found++; @@ -1987,10 +1987,8 @@ static bool gfs2_rgrp_used_recently(const struct gfs2_blkreserv *rs, static u32 gfs2_orlov_skip(const struct gfs2_inode *ip) { const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); - u32 skip; - get_random_bytes(&skip, sizeof(skip)); - return skip % sdp->sd_rgrps; + return get_random_u32() % sdp->sd_rgrps; } static bool gfs2_select_rgrp(struct gfs2_rgrpd **pos, const struct gfs2_rgrpd *begin) diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 6678060ed4d2..92a3b6ddafdc 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -44,10 +44,10 @@ #include "xattr.h" #include "lops.h" -enum dinode_demise { - SHOULD_DELETE_DINODE, - SHOULD_NOT_DELETE_DINODE, - SHOULD_DEFER_EVICTION, +enum evict_behavior { + EVICT_SHOULD_DELETE, + EVICT_SHOULD_SKIP_DELETE, + EVICT_SHOULD_DEFER_DELETE, }; /** @@ -1030,7 +1030,7 @@ static int gfs2_drop_inode(struct inode *inode) if (inode->i_nlink && gfs2_holder_initialized(&ip->i_iopen_gh)) { struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl; - if (test_bit(GLF_DEMOTE, &gl->gl_flags)) + if (glock_needs_demote(gl)) clear_nlink(inode); } @@ -1045,7 +1045,7 @@ static int gfs2_drop_inode(struct inode *inode) struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl; gfs2_glock_hold(gl); - if (!gfs2_queue_try_to_evict(gl)) + if (!gfs2_queue_verify_delete(gl, true)) gfs2_glock_put_async(gl); return 0; } @@ -1257,7 +1257,7 @@ static void gfs2_glock_put_eventually(struct gfs2_glock *gl) gfs2_glock_put(gl); } -static bool gfs2_upgrade_iopen_glock(struct inode *inode) +static enum evict_behavior gfs2_upgrade_iopen_glock(struct inode *inode) { struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); @@ -1272,9 +1272,9 @@ static bool gfs2_upgrade_iopen_glock(struct inode *inode) * exclusive access to the iopen glock here. * * Otherwise, the other nodes holding the lock will be notified about - * our locking request. If they do not have the inode open, they are - * expected to evict the cached inode and release the lock, allowing us - * to proceed. + * our locking request (see iopen_go_callback()). If they do not have + * the inode open, they are expected to evict the cached inode and + * release the lock, allowing us to proceed. * * Otherwise, if they cannot evict the inode, they are expected to poke * the inode glock (note: not the iopen glock). We will notice that @@ -1290,17 +1290,22 @@ static bool gfs2_upgrade_iopen_glock(struct inode *inode) gfs2_holder_reinit(LM_ST_EXCLUSIVE, GL_ASYNC | GL_NOCACHE, gh); error = gfs2_glock_nq(gh); if (error) - return false; + return EVICT_SHOULD_SKIP_DELETE; wait_event_interruptible_timeout(sdp->sd_async_glock_wait, !test_bit(HIF_WAIT, &gh->gh_iflags) || - test_bit(GLF_DEMOTE, &ip->i_gl->gl_flags), + glock_needs_demote(ip->i_gl), 5 * HZ); if (!test_bit(HIF_HOLDER, &gh->gh_iflags)) { gfs2_glock_dq(gh); - return false; + if (glock_needs_demote(ip->i_gl)) + return EVICT_SHOULD_SKIP_DELETE; + return EVICT_SHOULD_DEFER_DELETE; } - return gfs2_glock_holder_ready(gh) == 0; + error = gfs2_glock_holder_ready(gh); + if (error) + return EVICT_SHOULD_SKIP_DELETE; + return EVICT_SHOULD_DELETE; } /** @@ -1313,8 +1318,8 @@ static bool gfs2_upgrade_iopen_glock(struct inode *inode) * * Returns: the fate of the dinode */ -static enum dinode_demise evict_should_delete(struct inode *inode, - struct gfs2_holder *gh) +static enum evict_behavior evict_should_delete(struct inode *inode, + struct gfs2_holder *gh) { struct gfs2_inode *ip = GFS2_I(inode); struct super_block *sb = inode->i_sb; @@ -1324,12 +1329,12 @@ static enum dinode_demise evict_should_delete(struct inode *inode, if (unlikely(test_bit(GIF_ALLOC_FAILED, &ip->i_flags))) goto should_delete; - if (test_bit(GIF_DEFERRED_DELETE, &ip->i_flags)) - return SHOULD_DEFER_EVICTION; + if (test_bit(GIF_DEFER_DELETE, &ip->i_flags)) + return EVICT_SHOULD_DEFER_DELETE; /* Deletes should never happen under memory pressure anymore. */ if (WARN_ON_ONCE(current->flags & PF_MEMALLOC)) - return SHOULD_DEFER_EVICTION; + return EVICT_SHOULD_DEFER_DELETE; /* Must not read inode block until block type has been verified */ ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, gh); @@ -1337,34 +1342,37 @@ static enum dinode_demise evict_should_delete(struct inode *inode, glock_clear_object(ip->i_iopen_gh.gh_gl, ip); ip->i_iopen_gh.gh_flags |= GL_NOCACHE; gfs2_glock_dq_uninit(&ip->i_iopen_gh); - return SHOULD_DEFER_EVICTION; + return EVICT_SHOULD_DEFER_DELETE; } if (gfs2_inode_already_deleted(ip->i_gl, ip->i_no_formal_ino)) - return SHOULD_NOT_DELETE_DINODE; + return EVICT_SHOULD_SKIP_DELETE; ret = gfs2_check_blk_type(sdp, ip->i_no_addr, GFS2_BLKST_UNLINKED); if (ret) - return SHOULD_NOT_DELETE_DINODE; + return EVICT_SHOULD_SKIP_DELETE; ret = gfs2_instantiate(gh); if (ret) - return SHOULD_NOT_DELETE_DINODE; + return EVICT_SHOULD_SKIP_DELETE; /* * The inode may have been recreated in the meantime. */ if (inode->i_nlink) - return SHOULD_NOT_DELETE_DINODE; + return EVICT_SHOULD_SKIP_DELETE; should_delete: if (gfs2_holder_initialized(&ip->i_iopen_gh) && test_bit(HIF_HOLDER, &ip->i_iopen_gh.gh_iflags)) { - if (!gfs2_upgrade_iopen_glock(inode)) { + enum evict_behavior behavior = + gfs2_upgrade_iopen_glock(inode); + + if (behavior != EVICT_SHOULD_DELETE) { gfs2_holder_uninit(&ip->i_iopen_gh); - return SHOULD_NOT_DELETE_DINODE; + return behavior; } } - return SHOULD_DELETE_DINODE; + return EVICT_SHOULD_DELETE; } /** @@ -1475,8 +1483,10 @@ static void gfs2_evict_inode(struct inode *inode) struct gfs2_sbd *sdp = sb->s_fs_info; struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_holder gh; + enum evict_behavior behavior; int ret; + gfs2_holder_mark_uninitialized(&gh); if (inode->i_nlink || sb_rdonly(sb) || !ip->i_no_addr) goto out; @@ -1488,11 +1498,20 @@ static void gfs2_evict_inode(struct inode *inode) if (!sdp->sd_jdesc) goto out; - gfs2_holder_mark_uninitialized(&gh); - ret = evict_should_delete(inode, &gh); - if (ret == SHOULD_DEFER_EVICTION) - goto out; - if (ret == SHOULD_DELETE_DINODE) + behavior = evict_should_delete(inode, &gh); + if (behavior == EVICT_SHOULD_DEFER_DELETE && + !test_bit(SDF_KILL, &sdp->sd_flags)) { + struct gfs2_glock *io_gl = ip->i_iopen_gh.gh_gl; + + if (io_gl) { + gfs2_glock_hold(io_gl); + if (!gfs2_queue_verify_delete(io_gl, true)) + gfs2_glock_put(io_gl); + goto out; + } + behavior = EVICT_SHOULD_DELETE; + } + if (behavior == EVICT_SHOULD_DELETE) ret = evict_unlinked_inode(inode); else ret = evict_linked_inode(inode); @@ -1500,11 +1519,11 @@ static void gfs2_evict_inode(struct inode *inode) if (gfs2_rs_active(&ip->i_res)) gfs2_rs_deltree(&ip->i_res); - if (gfs2_holder_initialized(&gh)) - gfs2_glock_dq_uninit(&gh); if (ret && ret != GLR_TRYFAILED && ret != -EROFS) fs_warn(sdp, "gfs2_evict_inode: %d\n", ret); out: + if (gfs2_holder_initialized(&gh)) + gfs2_glock_dq_uninit(&gh); truncate_inode_pages_final(&inode->i_data); if (ip->i_qadata) gfs2_assert_warn(sdp, ip->i_qadata->qa_ref == 0); @@ -1537,11 +1556,13 @@ static struct inode *gfs2_alloc_inode(struct super_block *sb) if (!ip) return NULL; ip->i_no_addr = 0; + ip->i_no_formal_ino = 0; ip->i_flags = 0; ip->i_gl = NULL; gfs2_holder_mark_uninitialized(&ip->i_iopen_gh); memset(&ip->i_res, 0, sizeof(ip->i_res)); RB_CLEAR_NODE(&ip->i_res.rs_node); + ip->i_diskflags = 0; ip->i_rahead = 0; return &ip->i_inode; } diff --git a/fs/lockd/clntxdr.c b/fs/lockd/clntxdr.c index a3e97278b997..6ea3448d2d31 100644 --- a/fs/lockd/clntxdr.c +++ b/fs/lockd/clntxdr.c @@ -2,8 +2,9 @@ /* * linux/fs/lockd/clntxdr.c * - * XDR functions to encode/decode NLM version 3 RPC arguments and results. - * NLM version 3 is backwards compatible with NLM versions 1 and 2. + * XDR functions to encode/decode NLM version 1 and 3 RPC + * arguments and results. NLM version 2 is not specified + * by a standard, thus it is not implemented. * * NLM client-side only. * diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 8a72c418cdcc..109e5caae8c7 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -46,14 +46,15 @@ nlm4svc_retrieve_args(struct svc_rqst *rqstp, struct nlm_args *argp, if (filp != NULL) { int mode = lock_to_openmode(&lock->fl); + lock->fl.c.flc_flags = FL_POSIX; + error = nlm_lookup_file(rqstp, &file, lock); if (error) goto no_locks; *filp = file; /* Set up the missing parts of the file_lock structure */ - lock->fl.c.flc_flags = FL_POSIX; - lock->fl.c.flc_file = file->f_file[mode]; + lock->fl.c.flc_file = file->f_file[mode]; lock->fl.c.flc_pid = current->tgid; lock->fl.fl_start = (loff_t)lock->lock_start; lock->fl.fl_end = lock->lock_len ? @@ -108,7 +109,8 @@ __nlm4svc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) test_owner = argp->lock.fl.c.flc_owner; /* Now check for conflicting locks */ - resp->status = nlmsvc_testlock(rqstp, file, host, &argp->lock, &resp->lock, &resp->cookie); + resp->status = nlmsvc_testlock(rqstp, file, host, &argp->lock, + &resp->lock); if (resp->status == nlm_drop_reply) rc = rpc_drop_reply; else @@ -142,18 +144,6 @@ __nlm4svc_proc_lock(struct svc_rqst *rqstp, struct nlm_res *resp) if ((resp->status = nlm4svc_retrieve_args(rqstp, argp, &host, &file))) return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; -#if 0 - /* If supplied state doesn't match current state, we assume it's - * an old request that time-warped somehow. Any error return would - * do in this case because it's irrelevant anyway. - * - * NB: We don't retrieve the remote host's state yet. - */ - if (host->h_nsmstate && host->h_nsmstate != argp->state) { - resp->status = nlm_lck_denied_nolocks; - } else -#endif - /* Now try to lock the file */ resp->status = nlmsvc_lock(rqstp, file, host, &argp->lock, argp->block, &argp->cookie, diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index 2359347c9fbd..c1315df4b350 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -608,7 +608,7 @@ out: __be32 nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, struct nlm_host *host, struct nlm_lock *lock, - struct nlm_lock *conflock, struct nlm_cookie *cookie) + struct nlm_lock *conflock) { int error; int mode; diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index a03220e66ce0..f53d5177f267 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -130,7 +130,8 @@ __nlmsvc_proc_test(struct svc_rqst *rqstp, struct nlm_res *resp) test_owner = argp->lock.fl.c.flc_owner; /* Now check for conflicting locks */ - resp->status = cast_status(nlmsvc_testlock(rqstp, file, host, &argp->lock, &resp->lock, &resp->cookie)); + resp->status = cast_status(nlmsvc_testlock(rqstp, file, host, + &argp->lock, &resp->lock)); if (resp->status == nlm_drop_reply) rc = rpc_drop_reply; else @@ -165,18 +166,6 @@ __nlmsvc_proc_lock(struct svc_rqst *rqstp, struct nlm_res *resp) if ((resp->status = nlmsvc_retrieve_args(rqstp, argp, &host, &file))) return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; -#if 0 - /* If supplied state doesn't match current state, we assume it's - * an old request that time-warped somehow. Any error return would - * do in this case because it's irrelevant anyway. - * - * NB: We don't retrieve the remote host's state yet. - */ - if (host->h_nsmstate && host->h_nsmstate != argp->state) { - resp->status = nlm_lck_denied_nolocks; - } else -#endif - /* Now try to lock the file */ resp->status = cast_status(nlmsvc_lock(rqstp, file, host, &argp->lock, argp->block, &argp->cookie, diff --git a/fs/lockd/xdr4.c b/fs/lockd/xdr4.c index 3d28b9c3ed15..e343c820301f 100644 --- a/fs/lockd/xdr4.c +++ b/fs/lockd/xdr4.c @@ -89,7 +89,6 @@ svcxdr_decode_lock(struct xdr_stream *xdr, struct nlm_lock *lock) return false; locks_init_lock(fl); - fl->c.flc_flags = FL_POSIX; fl->c.flc_type = F_RDLCK; nlm4svc_set_file_lock_range(fl, lock->lock_start, lock->lock_len); return true; @@ -268,7 +267,6 @@ nlm4svc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) struct nlm_args *argp = rqstp->rq_argp; struct nlm_lock *lock = &argp->lock; - memset(lock, 0, sizeof(*lock)); locks_init_lock(&lock->fl); lock->svid = ~(u32)0; diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index 09404d142d1a..a74ec08f6c96 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -155,11 +155,9 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, /* We have an implied reference to net thanks to nfsd_serv_try_get */ localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred, nfs_fh, fmode); - if (IS_ERR(localio)) { - rcu_read_lock(); - nfs_to->nfsd_serv_put(net); - rcu_read_unlock(); - } + if (IS_ERR(localio)) + nfs_to_nfsd_net_put(net); + return localio; } EXPORT_SYMBOL_GPL(nfs_open_local_fh); diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index c82d8e3e0d4f..eacafe46e3b6 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -40,15 +40,24 @@ #define EXPKEY_HASHMAX (1 << EXPKEY_HASHBITS) #define EXPKEY_HASHMASK (EXPKEY_HASHMAX -1) -static void expkey_put(struct kref *ref) +static void expkey_put_work(struct work_struct *work) { - struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + struct svc_expkey *key = + container_of(to_rcu_work(work), struct svc_expkey, ek_rcu_work); if (test_bit(CACHE_VALID, &key->h.flags) && !test_bit(CACHE_NEGATIVE, &key->h.flags)) path_put(&key->ek_path); auth_domain_put(key->ek_client); - kfree_rcu(key, ek_rcu); + kfree(key); +} + +static void expkey_put(struct kref *ref) +{ + struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + + INIT_RCU_WORK(&key->ek_rcu_work, expkey_put_work); + queue_rcu_work(system_wq, &key->ek_rcu_work); } static int expkey_upcall(struct cache_detail *cd, struct cache_head *h) @@ -355,16 +364,26 @@ static void export_stats_destroy(struct export_stats *stats) EXP_STATS_COUNTERS_NUM); } -static void svc_export_put(struct kref *ref) +static void svc_export_put_work(struct work_struct *work) { - struct svc_export *exp = container_of(ref, struct svc_export, h.ref); + struct svc_export *exp = + container_of(to_rcu_work(work), struct svc_export, ex_rcu_work); + path_put(&exp->ex_path); auth_domain_put(exp->ex_client); nfsd4_fslocs_free(&exp->ex_fslocs); export_stats_destroy(exp->ex_stats); kfree(exp->ex_stats); kfree(exp->ex_uuid); - kfree_rcu(exp, ex_rcu); + kfree(exp); +} + +static void svc_export_put(struct kref *ref) +{ + struct svc_export *exp = container_of(ref, struct svc_export, h.ref); + + INIT_RCU_WORK(&exp->ex_rcu_work, svc_export_put_work); + queue_rcu_work(system_wq, &exp->ex_rcu_work); } static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h) @@ -1078,12 +1097,14 @@ static struct svc_export *exp_find(struct cache_detail *cd, * check_nfsd_access - check if access to export is allowed. * @exp: svc_export that is being accessed. * @rqstp: svc_rqst attempting to access @exp (will be NULL for LOCALIO). + * @may_bypass_gss: reduce strictness of authorization check * * Return values: * %nfs_ok if access is granted, or * %nfserr_wrongsec if access is denied */ -__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp) +__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, + bool may_bypass_gss) { struct exp_flavor_info *f, *end = exp->ex_flavors + exp->ex_nflavors; struct svc_xprt *xprt; @@ -1140,6 +1161,23 @@ ok: if (nfsd4_spo_must_allow(rqstp)) return nfs_ok; + /* Some calls may be processed without authentication + * on GSS exports. For example NFS2/3 calls on root + * directory, see section 2.3.2 of rfc 2623. + * For "may_bypass_gss" check that export has really + * enabled some flavor with authentication (GSS or any + * other) and also check that the used auth flavor is + * without authentication (none or sys). + */ + if (may_bypass_gss && ( + rqstp->rq_cred.cr_flavor == RPC_AUTH_NULL || + rqstp->rq_cred.cr_flavor == RPC_AUTH_UNIX)) { + for (f = exp->ex_flavors; f < end; f++) { + if (f->pseudoflavor >= RPC_AUTH_DES) + return 0; + } + } + denied: return nfserr_wrongsec; } @@ -1406,9 +1444,12 @@ static int e_show(struct seq_file *m, void *p) return 0; } - exp_get(exp); + if (!cache_get_rcu(&exp->h)) + return 0; + if (cache_check(cd, &exp->h, NULL)) return 0; + exp_put(exp); return svc_export_show(m, cd, cp); } diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index 3794ae253a70..6f2fbaae01fa 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -75,7 +75,7 @@ struct svc_export { u32 ex_layout_types; struct nfsd4_deviceid_map *ex_devid_map; struct cache_detail *cd; - struct rcu_head ex_rcu; + struct rcu_work ex_rcu_work; unsigned long ex_xprtsec_modes; struct export_stats *ex_stats; }; @@ -92,7 +92,7 @@ struct svc_expkey { u32 ek_fsid[6]; struct path ek_path; - struct rcu_head ek_rcu; + struct rcu_work ek_rcu_work; }; #define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) @@ -101,7 +101,8 @@ struct svc_expkey { struct svc_cred; int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp); -__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp); +__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, + bool may_bypass_gss); /* * Function declarations diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 09c444eb944f..a1cdba42c4fa 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -391,19 +391,19 @@ nfsd_file_put(struct nfsd_file *nf) } /** - * nfsd_file_put_local - put the reference to nfsd_file and local nfsd_serv - * @nf: nfsd_file of which to put the references + * nfsd_file_put_local - put nfsd_file reference and arm nfsd_serv_put in caller + * @nf: nfsd_file of which to put the reference * - * First put the reference of the nfsd_file and then put the - * reference to the associated nn->nfsd_serv. + * First save the associated net to return to caller, then put + * the reference of the nfsd_file. */ -void -nfsd_file_put_local(struct nfsd_file *nf) __must_hold(rcu) +struct net * +nfsd_file_put_local(struct nfsd_file *nf) { struct net *net = nf->nf_net; nfsd_file_put(nf); - nfsd_serv_put(net); + return net; } /** @@ -1047,7 +1047,7 @@ retry: * the last one however, since we should hold another. */ if (nfsd_file_lru_remove(nf)) - WARN_ON_ONCE(refcount_dec_and_test(&nf->nf_ref)); + refcount_dec(&nf->nf_ref); goto wait_for_construction; } @@ -1120,8 +1120,7 @@ open_file: status = nfs_ok; trace_nfsd_file_opened(nf, status); } else { - ret = nfsd_open_verified(rqstp, fhp, may_flags, - &nf->nf_file); + ret = nfsd_open_verified(fhp, may_flags, &nf->nf_file); if (ret == -EOPENSTALE && stale_retry) { stale_retry = false; nfsd_file_unhash(nf); diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index cadf3c2689c4..d5db6b34ba30 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -55,7 +55,7 @@ void nfsd_file_cache_shutdown(void); int nfsd_file_cache_start_net(struct net *net); void nfsd_file_cache_shutdown_net(struct net *net); void nfsd_file_put(struct nfsd_file *nf); -void nfsd_file_put_local(struct nfsd_file *nf); +struct net *nfsd_file_put_local(struct nfsd_file *nf); struct nfsd_file *nfsd_file_get(struct nfsd_file *nf); struct file *nfsd_file_file(struct nfsd_file *nf); void nfsd_file_close_inode_sync(struct inode *inode); diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c index 46a7f9b813e5..edc9f75dc75c 100644 --- a/fs/nfsd/lockd.c +++ b/fs/nfsd/lockd.c @@ -38,11 +38,20 @@ nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp, memcpy(&fh.fh_handle.fh_raw, f->data, f->size); fh.fh_export = NULL; + /* + * Allow BYPASS_GSS as some client implementations use AUTH_SYS + * for NLM even when GSS is used for NFS. + * Allow OWNER_OVERRIDE as permission might have been changed + * after the file was opened. + * Pass MAY_NLM so that authentication can be completely bypassed + * if NFSEXP_NOAUTHNLM is set. Some older clients use AUTH_NULL + * for NLM requests. + */ access = (mode == O_WRONLY) ? NFSD_MAY_WRITE : NFSD_MAY_READ; - access |= NFSD_MAY_LOCK; + access |= NFSD_MAY_NLM | NFSD_MAY_OWNER_OVERRIDE | NFSD_MAY_BYPASS_GSS; nfserr = nfsd_open(rqstp, &fh, S_IFREG, access, filp); fh_put(&fh); - /* We return nlm error codes as nlm doesn't know + /* We return nlm error codes as nlm doesn't know * about nfsd, but nfsd does know about nlm.. */ switch (nfserr) { diff --git a/fs/nfsd/nfs4acl.c b/fs/nfsd/nfs4acl.c index 96e786b5e544..936ea1ad9586 100644 --- a/fs/nfsd/nfs4acl.c +++ b/fs/nfsd/nfs4acl.c @@ -198,8 +198,6 @@ summarize_posix_acl(struct posix_acl *acl, struct posix_acl_summary *pas) memset(pas, 0, sizeof(*pas)); pas->mask = 07; - pe = acl->a_entries + acl->a_count; - FOREACH_ACL_ENTRY(pa, acl, pe) { switch (pa->e_tag) { case ACL_USER_OBJ: diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index b5b3ab9d719a..3877b53e429f 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -287,17 +287,17 @@ static int decode_cb_compound4res(struct xdr_stream *xdr, u32 length; __be32 *p; - p = xdr_inline_decode(xdr, 4 + 4); + p = xdr_inline_decode(xdr, XDR_UNIT); if (unlikely(p == NULL)) goto out_overflow; - hdr->status = be32_to_cpup(p++); + hdr->status = be32_to_cpup(p); /* Ignore the tag */ - length = be32_to_cpup(p++); - p = xdr_inline_decode(xdr, length + 4); - if (unlikely(p == NULL)) + if (xdr_stream_decode_u32(xdr, &length) < 0) + goto out_overflow; + if (xdr_inline_decode(xdr, length) == NULL) + goto out_overflow; + if (xdr_stream_decode_u32(xdr, &hdr->nops) < 0) goto out_overflow; - p += XDR_QUADLEN(length); - hdr->nops = be32_to_cpup(p); return 0; out_overflow: return -EIO; @@ -364,13 +364,29 @@ encode_cb_getattr4args(struct xdr_stream *xdr, struct nfs4_cb_compound_hdr *hdr, struct nfs4_delegation *dp = container_of(fattr, struct nfs4_delegation, dl_cb_fattr); struct knfsd_fh *fh = &dp->dl_stid.sc_file->fi_fhandle; + u32 bmap[1]; + + bmap[0] = FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE; encode_nfs_cb_opnum4(xdr, OP_CB_GETATTR); encode_nfs_fh4(xdr, fh); - encode_bitmap4(xdr, fattr->ncf_cb_bmap, ARRAY_SIZE(fattr->ncf_cb_bmap)); + encode_bitmap4(xdr, bmap, ARRAY_SIZE(bmap)); hdr->nops++; } +static u32 highest_slotid(struct nfsd4_session *ses) +{ + u32 idx; + + spin_lock(&ses->se_lock); + idx = fls(~ses->se_cb_slot_avail); + if (idx > 0) + --idx; + idx = max(idx, ses->se_cb_highest_slot); + spin_unlock(&ses->se_lock); + return idx; +} + /* * CB_SEQUENCE4args * @@ -397,15 +413,40 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, encode_sessionid4(xdr, session); p = xdr_reserve_space(xdr, 4 + 4 + 4 + 4 + 4); - *p++ = cpu_to_be32(session->se_cb_seq_nr); /* csa_sequenceid */ - *p++ = xdr_zero; /* csa_slotid */ - *p++ = xdr_zero; /* csa_highest_slotid */ + *p++ = cpu_to_be32(session->se_cb_seq_nr[cb->cb_held_slot]); /* csa_sequenceid */ + *p++ = cpu_to_be32(cb->cb_held_slot); /* csa_slotid */ + *p++ = cpu_to_be32(highest_slotid(session)); /* csa_highest_slotid */ *p++ = xdr_zero; /* csa_cachethis */ xdr_encode_empty_array(p); /* csa_referring_call_lists */ hdr->nops++; } +static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) +{ + /* No need to do anything if nothing changed */ + if (likely(target == READ_ONCE(ses->se_cb_highest_slot))) + return; + + spin_lock(&ses->se_lock); + if (target > ses->se_cb_highest_slot) { + int i; + + target = min(target, NFSD_BC_SLOT_TABLE_SIZE - 1); + + /* + * Growing the slot table. Reset any new sequences to 1. + * + * NB: There is some debate about whether the RFC requires this, + * but the Linux client expects it. + */ + for (i = ses->se_cb_highest_slot + 1; i <= target; ++i) + ses->se_cb_seq_nr[i] = 1; + } + ses->se_cb_highest_slot = target; + spin_unlock(&ses->se_lock); +} + /* * CB_SEQUENCE4resok * @@ -433,7 +474,7 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, struct nfsd4_session *session = cb->cb_clp->cl_cb_session; int status = -ESERVERFAULT; __be32 *p; - u32 dummy; + u32 seqid, slotid, target; /* * If the server returns different values for sessionID, slotID or @@ -449,21 +490,22 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, } p += XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN); - dummy = be32_to_cpup(p++); - if (dummy != session->se_cb_seq_nr) { + seqid = be32_to_cpup(p++); + if (seqid != session->se_cb_seq_nr[cb->cb_held_slot]) { dprintk("NFS: %s Invalid sequence number\n", __func__); goto out; } - dummy = be32_to_cpup(p++); - if (dummy != 0) { + slotid = be32_to_cpup(p++); + if (slotid != cb->cb_held_slot) { dprintk("NFS: %s Invalid slotid\n", __func__); goto out; } - /* - * FIXME: process highest slotid and target highest slotid - */ + p++; // ignore current highest slot value + + target = be32_to_cpup(p++); + update_cb_slot_table(session, target); status = 0; out: cb->cb_seq_status = status; @@ -1164,6 +1206,22 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) spin_unlock(&clp->cl_lock); } +static int grab_slot(struct nfsd4_session *ses) +{ + int idx; + + spin_lock(&ses->se_lock); + idx = ffs(ses->se_cb_slot_avail) - 1; + if (idx < 0 || idx > ses->se_cb_highest_slot) { + spin_unlock(&ses->se_lock); + return -1; + } + /* clear the bit for the slot */ + ses->se_cb_slot_avail &= ~BIT(idx); + spin_unlock(&ses->se_lock); + return idx; +} + /* * There's currently a single callback channel slot. * If the slot is available, then mark it busy. Otherwise, set the @@ -1172,28 +1230,32 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) { struct nfs4_client *clp = cb->cb_clp; + struct nfsd4_session *ses = clp->cl_cb_session; - if (!cb->cb_holds_slot && - test_and_set_bit(0, &clp->cl_cb_slot_busy) != 0) { + if (cb->cb_held_slot >= 0) + return true; + cb->cb_held_slot = grab_slot(ses); + if (cb->cb_held_slot < 0) { rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); /* Race breaker */ - if (test_and_set_bit(0, &clp->cl_cb_slot_busy) != 0) { - dprintk("%s slot is busy\n", __func__); + cb->cb_held_slot = grab_slot(ses); + if (cb->cb_held_slot < 0) return false; - } rpc_wake_up_queued_task(&clp->cl_cb_waitq, task); } - cb->cb_holds_slot = true; return true; } static void nfsd41_cb_release_slot(struct nfsd4_callback *cb) { struct nfs4_client *clp = cb->cb_clp; + struct nfsd4_session *ses = clp->cl_cb_session; - if (cb->cb_holds_slot) { - cb->cb_holds_slot = false; - clear_bit(0, &clp->cl_cb_slot_busy); + if (cb->cb_held_slot >= 0) { + spin_lock(&ses->se_lock); + ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); + spin_unlock(&ses->se_lock); + cb->cb_held_slot = -1; rpc_wake_up_next(&clp->cl_cb_waitq); } } @@ -1210,8 +1272,8 @@ static void nfsd41_destroy_cb(struct nfsd4_callback *cb) } /* - * TODO: cb_sequence should support referring call lists, cachethis, multiple - * slots, and mark callback channel down on communication errors. + * TODO: cb_sequence should support referring call lists, cachethis, + * and mark callback channel down on communication errors. */ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata) { @@ -1253,7 +1315,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback return true; } - if (!cb->cb_holds_slot) + if (cb->cb_held_slot < 0) goto need_restart; /* This is the operation status code for CB_SEQUENCE */ @@ -1267,10 +1329,10 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback * If CB_SEQUENCE returns an error, then the state of the slot * (sequence ID, cached reply) MUST NOT change. */ - ++session->se_cb_seq_nr; + ++session->se_cb_seq_nr[cb->cb_held_slot]; break; case -ESERVERFAULT: - ++session->se_cb_seq_nr; + ++session->se_cb_seq_nr[cb->cb_held_slot]; nfsd4_mark_cb_fault(cb->cb_clp); ret = false; break; @@ -1296,17 +1358,16 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback case -NFS4ERR_BADSLOT: goto retry_nowait; case -NFS4ERR_SEQ_MISORDERED: - if (session->se_cb_seq_nr != 1) { - session->se_cb_seq_nr = 1; + if (session->se_cb_seq_nr[cb->cb_held_slot] != 1) { + session->se_cb_seq_nr[cb->cb_held_slot] = 1; goto retry_nowait; } break; default: nfsd4_mark_cb_fault(cb->cb_clp); } - nfsd41_cb_release_slot(cb); - trace_nfsd_cb_free_slot(task, cb); + nfsd41_cb_release_slot(cb); if (RPC_SIGNALLED(task)) goto need_restart; @@ -1461,6 +1522,8 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) ses = c->cn_session; } spin_unlock(&clp->cl_lock); + if (!c) + return; err = setup_callback_client(clp, &conn, ses); if (err) { @@ -1524,7 +1587,7 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, INIT_WORK(&cb->cb_work, nfsd4_run_cb_work); cb->cb_status = 0; cb->cb_need_restart = false; - cb->cb_holds_slot = false; + cb->cb_held_slot = -1; } /** diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index d32f2dfd148f..f8a10f90bc7a 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -57,6 +57,8 @@ module_param(inter_copy_offload_enable, bool, 0644); MODULE_PARM_DESC(inter_copy_offload_enable, "Enable inter server to server copy offload. Default: false"); +static void cleanup_async_copy(struct nfsd4_copy *copy); + #ifdef CONFIG_NFSD_V4_2_INTER_SSC static int nfsd4_ssc_umount_timeout = 900000; /* default to 15 mins */ module_param(nfsd4_ssc_umount_timeout, int, 0644); @@ -1276,6 +1278,71 @@ out: return status; } +/** + * nfsd4_has_active_async_copies - Check for ongoing copy operations + * @clp: Client to be checked + * + * NFSD maintains state for async COPY operations after they complete, + * and this state remains in the nfs4_client's async_copies list. + * Ongoing copies should block the destruction of the nfs4_client, but + * completed copies should not. + * + * Return values: + * %true: At least one active async COPY is ongoing + * %false: No active async COPY operations were found + */ +bool nfsd4_has_active_async_copies(struct nfs4_client *clp) +{ + struct nfsd4_copy *copy; + bool result = false; + + spin_lock(&clp->async_lock); + list_for_each_entry(copy, &clp->async_copies, copies) { + if (!test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags) && + !test_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags)) { + result = true; + break; + } + } + spin_unlock(&clp->async_lock); + return result; +} + +/** + * nfsd4_async_copy_reaper - Purge completed copies + * @nn: Network namespace with possible active copy information + */ +void nfsd4_async_copy_reaper(struct nfsd_net *nn) +{ + struct nfs4_client *clp; + struct nfsd4_copy *copy; + LIST_HEAD(reaplist); + + spin_lock(&nn->client_lock); + list_for_each_entry(clp, &nn->client_lru, cl_lru) { + struct list_head *pos, *next; + + spin_lock(&clp->async_lock); + list_for_each_safe(pos, next, &clp->async_copies) { + copy = list_entry(pos, struct nfsd4_copy, copies); + if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags)) { + if (--copy->cp_ttl) { + list_del_init(©->copies); + list_add(©->copies, &reaplist); + } + } + } + spin_unlock(&clp->async_lock); + } + spin_unlock(&nn->client_lock); + + while (!list_empty(&reaplist)) { + copy = list_first_entry(&reaplist, struct nfsd4_copy, copies); + list_del_init(©->copies); + cleanup_async_copy(copy); + } +} + static void nfs4_put_copy(struct nfsd4_copy *copy) { if (!refcount_dec_and_test(©->refcount)) @@ -1287,12 +1354,13 @@ static void nfs4_put_copy(struct nfsd4_copy *copy) static void nfsd4_stop_copy(struct nfsd4_copy *copy) { + trace_nfsd_copy_async_cancel(copy); if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags)) kthread_stop(copy->copy_task); nfs4_put_copy(copy); } -static struct nfsd4_copy *nfsd4_get_copy(struct nfs4_client *clp) +static struct nfsd4_copy *nfsd4_unhash_copy(struct nfs4_client *clp) { struct nfsd4_copy *copy = NULL; @@ -1301,6 +1369,9 @@ static struct nfsd4_copy *nfsd4_get_copy(struct nfs4_client *clp) copy = list_first_entry(&clp->async_copies, struct nfsd4_copy, copies); refcount_inc(©->refcount); + copy->cp_clp = NULL; + if (!list_empty(©->copies)) + list_del_init(©->copies); } spin_unlock(&clp->async_lock); return copy; @@ -1310,7 +1381,7 @@ void nfsd4_shutdown_copy(struct nfs4_client *clp) { struct nfsd4_copy *copy; - while ((copy = nfsd4_get_copy(clp)) != NULL) + while ((copy = nfsd4_unhash_copy(clp)) != NULL) nfsd4_stop_copy(copy); } #ifdef CONFIG_NFSD_V4_2_INTER_SSC @@ -1598,8 +1669,10 @@ static void nfsd4_cb_offload_release(struct nfsd4_callback *cb) { struct nfsd4_cb_offload *cbo = container_of(cb, struct nfsd4_cb_offload, co_cb); + struct nfsd4_copy *copy = + container_of(cbo, struct nfsd4_copy, cp_cb_offload); - kfree(cbo); + set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); } static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, @@ -1609,6 +1682,13 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, container_of(cb, struct nfsd4_cb_offload, co_cb); trace_nfsd_cb_offload_done(&cbo->co_res.cb_stateid, task); + switch (task->tk_status) { + case -NFS4ERR_DELAY: + if (cbo->co_retries--) { + rpc_delay(task, 1 * HZ); + return 0; + } + } return 1; } @@ -1732,15 +1812,12 @@ static void cleanup_async_copy(struct nfsd4_copy *copy) static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) { - struct nfsd4_cb_offload *cbo; - - cbo = kzalloc(sizeof(*cbo), GFP_KERNEL); - if (!cbo) - return; + struct nfsd4_cb_offload *cbo = ©->cp_cb_offload; memcpy(&cbo->co_res, ©->cp_res, sizeof(copy->cp_res)); memcpy(&cbo->co_fh, ©->fh, sizeof(copy->fh)); cbo->co_nfserr = copy->nfserr; + cbo->co_retries = 5; nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD); @@ -1786,10 +1863,13 @@ static int nfsd4_do_async_copy(void *data) } do_callback: + /* The kthread exits forthwith. Ensure that a subsequent + * OFFLOAD_CANCEL won't try to kill it again. */ + set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags); + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); trace_nfsd_copy_async_done(copy); nfsd4_send_cb_offload(copy); - cleanup_async_copy(copy); return 0; } @@ -1843,6 +1923,7 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, async_copy->cp_nn = nn; INIT_LIST_HEAD(&async_copy->copies); refcount_set(&async_copy->refcount, 1); + async_copy->cp_ttl = NFSD_COPY_INITIAL_TTL; /* Arbitrary cap on number of pending async copy operations */ if (atomic_inc_return(&nn->pending_async_copies) > (int)rqstp->rq_pool->sp_nrthreads) @@ -2780,6 +2861,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) if (op->opdesc->op_get_currentstateid) op->opdesc->op_get_currentstateid(cstate, &op->u); op->status = op->opdesc->op_func(rqstp, cstate, &op->u); + trace_nfsd_compound_op_err(rqstp, op->opnum, op->status); /* Only from SEQUENCE */ if (cstate->status == nfserr_replay_cache) { @@ -2796,7 +2878,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) if (current_fh->fh_export && need_wrongsec_check(rqstp)) - op->status = check_nfsd_access(current_fh->fh_export, rqstp); + op->status = check_nfsd_access(current_fh->fh_export, rqstp, false); } encode_op: if (op->status == nfserr_replay_me) { @@ -3452,6 +3534,7 @@ static const struct nfsd4_operation nfsd4_ops[] = { /* NFSv4.1 operations */ [OP_EXCHANGE_ID] = { .op_func = nfsd4_exchange_id, + .op_release = nfsd4_exchange_id_release, .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP | OP_MODIFIES_SOMETHING, .op_name = "OP_EXCHANGE_ID", diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index b7d61eb8afe9..4a765555bf84 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -659,7 +659,8 @@ nfs4_reset_recoverydir(char *recdir) return status; status = -ENOTDIR; if (d_is_dir(path.dentry)) { - strcpy(user_recovery_dirname, recdir); + strscpy(user_recovery_dirname, recdir, + sizeof(user_recovery_dirname)); status = 0; } path_put(&path); diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index d80406f8b568..741b9449f727 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -149,14 +149,14 @@ void nfsd4_destroy_laundry_wq(void) static bool is_session_dead(struct nfsd4_session *ses) { - return ses->se_flags & NFS4_SESSION_DEAD; + return ses->se_dead; } static __be32 mark_session_dead_locked(struct nfsd4_session *ses, int ref_held_by_me) { if (atomic_read(&ses->se_ref) > ref_held_by_me) return nfserr_jukebox; - ses->se_flags |= NFS4_SESSION_DEAD; + ses->se_dead = true; return nfs_ok; } @@ -572,13 +572,6 @@ opaque_hashval(const void *ptr, int nbytes) return x; } -static void nfsd4_free_file_rcu(struct rcu_head *rcu) -{ - struct nfs4_file *fp = container_of(rcu, struct nfs4_file, fi_rcu); - - kmem_cache_free(file_slab, fp); -} - void put_nfs4_file(struct nfs4_file *fi) { @@ -586,7 +579,7 @@ put_nfs4_file(struct nfs4_file *fi) nfsd4_file_hash_remove(fi); WARN_ON_ONCE(!list_empty(&fi->fi_clnt_odstate)); WARN_ON_ONCE(!list_empty(&fi->fi_delegations)); - call_rcu(&fi->fi_rcu, nfsd4_free_file_rcu); + kfree_rcu(fi, fi_rcu); } } @@ -1184,7 +1177,6 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, nfsd4_init_cb(&dp->dl_cb_fattr.ncf_getattr, dp->dl_stid.sc_client, &nfsd4_cb_getattr_ops, NFSPROC4_CLNT_CB_GETATTR); dp->dl_cb_fattr.ncf_file_modified = false; - dp->dl_cb_fattr.ncf_cb_bmap[0] = FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE; get_nfs4_file(fp); dp->dl_stid.sc_file = fp; return dp; @@ -1660,6 +1652,14 @@ static void release_open_stateid(struct nfs4_ol_stateid *stp) free_ol_stateid_reaplist(&reaplist); } +static bool nfs4_openowner_unhashed(struct nfs4_openowner *oo) +{ + lockdep_assert_held(&oo->oo_owner.so_client->cl_lock); + + return list_empty(&oo->oo_owner.so_strhash) && + list_empty(&oo->oo_perclient); +} + static void unhash_openowner_locked(struct nfs4_openowner *oo) { struct nfs4_client *clp = oo->oo_owner.so_client; @@ -2010,8 +2010,10 @@ static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fattrs, } memcpy(&new->se_fchannel, fattrs, sizeof(struct nfsd4_channel_attrs)); - memcpy(&new->se_bchannel, battrs, sizeof(struct nfsd4_channel_attrs)); - + new->se_cb_slot_avail = ~0U; + new->se_cb_highest_slot = min(battrs->maxreqs - 1, + NFSD_BC_SLOT_TABLE_SIZE - 1); + spin_lock_init(&new->se_lock); return new; out_free: while (i--) @@ -2142,11 +2144,14 @@ static void init_session(struct svc_rqst *rqstp, struct nfsd4_session *new, stru INIT_LIST_HEAD(&new->se_conns); - new->se_cb_seq_nr = 1; - new->se_flags = cses->flags; + atomic_set(&new->se_ref, 0); + new->se_dead = false; new->se_cb_prog = cses->callback_prog; new->se_cb_sec = cses->cb_sec; - atomic_set(&new->se_ref, 0); + + for (idx = 0; idx < NFSD_BC_SLOT_TABLE_SIZE; ++idx) + new->se_cb_seq_nr[idx] = 1; + idx = hash_sessionid(&new->se_sessionid); list_add(&new->se_hash, &nn->sessionid_hashtbl[idx]); spin_lock(&clp->cl_lock); @@ -2239,21 +2244,16 @@ STALE_CLIENTID(clientid_t *clid, struct nfsd_net *nn) return 1; } -/* - * XXX Should we use a slab cache ? - * This type of memory management is somewhat inefficient, but we use it - * anyway since SETCLIENTID is not a common operation. - */ static struct nfs4_client *alloc_client(struct xdr_netobj name, struct nfsd_net *nn) { struct nfs4_client *clp; int i; - if (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients) { + if (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients && + atomic_read(&nn->nfsd_courtesy_clients) > 0) mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); - return NULL; - } + clp = kmem_cache_zalloc(client_slab, GFP_KERNEL); if (clp == NULL) return NULL; @@ -3160,7 +3160,6 @@ static struct nfs4_client *create_client(struct xdr_netobj name, kref_init(&clp->cl_nfsdfs.cl_ref); nfsd4_init_cb(&clp->cl_cb_null, clp, NULL, NFSPROC4_CLNT_CB_NULL); clp->cl_time = ktime_get_boottime_seconds(); - clear_bit(0, &clp->cl_cb_slot_busy); copy_verf(clp, verf); memcpy(&clp->cl_addr, sa, sizeof(struct sockaddr_storage)); clp->cl_cb_session = NULL; @@ -3487,7 +3486,7 @@ static bool client_has_state(struct nfs4_client *clp) #endif || !list_empty(&clp->cl_delegations) || !list_empty(&clp->cl_sessions) - || !list_empty(&clp->async_copies); + || nfsd4_has_active_async_copies(clp); } static __be32 copy_impl_id(struct nfs4_client *clp, @@ -3525,6 +3524,12 @@ nfsd4_exchange_id(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, __func__, rqstp, exid, exid->clname.len, exid->clname.data, addr_str, exid->flags, exid->spa_how); + exid->server_impl_name = kasprintf(GFP_KERNEL, "%s %s %s %s", + utsname()->sysname, utsname()->release, + utsname()->version, utsname()->machine); + if (!exid->server_impl_name) + return nfserr_jukebox; + if (exid->flags & ~EXCHGID4_FLAG_MASK_A) return nfserr_inval; @@ -3662,6 +3667,23 @@ out_copy: exid->seqid = conf->cl_cs_slot.sl_seqid + 1; nfsd4_set_ex_flags(conf, exid); + exid->nii_domain.len = sizeof("kernel.org") - 1; + exid->nii_domain.data = "kernel.org"; + + /* + * Note that RFC 8881 places no length limit on + * nii_name, but this implementation permits no + * more than NFS4_OPAQUE_LIMIT bytes. + */ + exid->nii_name.len = strlen(exid->server_impl_name); + if (exid->nii_name.len > NFS4_OPAQUE_LIMIT) + exid->nii_name.len = NFS4_OPAQUE_LIMIT; + exid->nii_name.data = exid->server_impl_name; + + /* just send zeros - the date is in nii_name */ + exid->nii_time.tv_sec = 0; + exid->nii_time.tv_nsec = 0; + dprintk("nfsd4_exchange_id seqid %d flags %x\n", conf->cl_cs_slot.sl_seqid, conf->cl_exchange_flags); status = nfs_ok; @@ -3678,6 +3700,14 @@ out_nolock: return status; } +void +nfsd4_exchange_id_release(union nfsd4_op_u *u) +{ + struct nfsd4_exchange_id *exid = &u->exchange_id; + + kfree(exid->server_impl_name); +} + static __be32 check_slot_seqid(u32 seqid, u32 slot_seqid, bool slot_inuse) { /* The slot is in use, and no response has been sent. */ @@ -3911,6 +3941,8 @@ nfsd4_create_session(struct svc_rqst *rqstp, cr_ses->flags &= ~SESSION4_PERSIST; /* Upshifting from TCP to RDMA is not supported */ cr_ses->flags &= ~SESSION4_RDMA; + /* Report the correct number of backchannel slots */ + cr_ses->back_channel.maxreqs = new->se_cb_highest_slot + 1; init_session(rqstp, new, conf, cr_ses); nfsd4_get_session_locked(new); @@ -3931,7 +3963,6 @@ nfsd4_create_session(struct svc_rqst *rqstp, return status; out_expired_error: - old = NULL; /* * Revert the slot seq_nr change so the server will process * the client's resend instead of returning a cached response. @@ -3946,8 +3977,6 @@ out_cache_error: out_free_conn: spin_unlock(&nn->client_lock); free_conn(conn); - if (old) - expire_client(old); out_free_session: __free_session(new); out_release_drc_mem: @@ -4975,6 +5004,12 @@ retry: spin_lock(&oo->oo_owner.so_client->cl_lock); spin_lock(&fp->fi_lock); + if (nfs4_openowner_unhashed(oo)) { + mutex_unlock(&stp->st_mutex); + stp = NULL; + goto out_unlock; + } + retstp = nfsd4_find_existing_open(fp, open); if (retstp) goto out_unlock; @@ -5957,7 +5992,7 @@ nfs4_delegation_stat(struct nfs4_delegation *dp, struct svc_fh *currentfh, path.dentry = file_dentry(nf->nf_file); rc = vfs_getattr(&path, stat, - (STATX_SIZE | STATX_CTIME | STATX_CHANGE_COOKIE), + (STATX_MODE | STATX_SIZE | STATX_CTIME | STATX_CHANGE_COOKIE), AT_STATX_SYNC_AS_STAT); nfsd_file_put(nf); @@ -6041,8 +6076,7 @@ nfs4_open_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, } open->op_delegate_type = NFS4_OPEN_DELEGATE_WRITE; dp->dl_cb_fattr.ncf_cur_fsize = stat.size; - dp->dl_cb_fattr.ncf_initial_cinfo = - nfsd4_change_attribute(&stat, d_inode(currentfh->fh_dentry)); + dp->dl_cb_fattr.ncf_initial_cinfo = nfsd4_change_attribute(&stat); trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); } else { open->op_delegate_type = NFS4_OPEN_DELEGATE_READ; @@ -6127,6 +6161,11 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf if (!stp) { stp = init_open_stateid(fp, open); + if (!stp) { + status = nfserr_jukebox; + goto out; + } + if (!open->op_stp) new_stp = true; } @@ -6562,6 +6601,7 @@ nfs4_laundromat(struct nfsd_net *nn) _free_cpntf_state_locked(nn, cps); } spin_unlock(&nn->s2s_cp_lock); + nfsd4_async_copy_reaper(nn); nfs4_get_client_reaplist(nn, &reaplist, <); nfs4_process_client_reaplist(&reaplist); @@ -7943,11 +7983,9 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (check_lock_length(lock->lk_offset, lock->lk_length)) return nfserr_inval; - if ((status = fh_verify(rqstp, &cstate->current_fh, - S_IFREG, NFSD_MAY_LOCK))) { - dprintk("NFSD: nfsd4_lock: permission denied!\n"); + status = fh_verify(rqstp, &cstate->current_fh, S_IFREG, 0); + if (status != nfs_ok) return status; - } sb = cstate->current_fh.fh_dentry->d_sb; if (lock->lk_is_new) { @@ -8854,8 +8892,7 @@ nfsd4_get_writestateid(struct nfsd4_compound_state *cstate, * nfsd4_deleg_getattr_conflict - Recall if GETATTR causes conflict * @rqstp: RPC transaction context * @dentry: dentry of inode to be checked for a conflict - * @modified: return true if file was modified - * @size: new size of file if modified is true + * @pdp: returned WRITE delegation, if one was found * * This function is called when there is a conflict between a write * delegation and a change/size GETATTR from another client. The server @@ -8865,11 +8902,12 @@ nfsd4_get_writestateid(struct nfsd4_compound_state *cstate, * 18.7.4. * * Returns 0 if there is no conflict; otherwise an nfs_stat - * code is returned. + * code is returned. If @pdp is set to a non-NULL value, then the + * caller must put the reference. */ __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, - bool *modified, u64 *size) + struct nfs4_delegation **pdp) { __be32 status; struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); @@ -8880,10 +8918,9 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, struct nfs4_cb_fattr *ncf; struct inode *inode = d_inode(dentry); - *modified = false; ctx = locks_inode_context(inode); if (!ctx) - return 0; + return nfs_ok; #define NON_NFSD_LEASE ((void *)1) @@ -8949,10 +8986,10 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, goto out_status; } ncf->ncf_cur_fsize = ncf->ncf_cb_fsize; - *size = ncf->ncf_cur_fsize; - *modified = true; + *pdp = dp; + return nfs_ok; } - status = 0; + status = nfs_ok; out_status: nfs4_put_stid(&dp->dl_stid); return status; diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index f118921250c3..53fac037611c 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -2652,13 +2652,10 @@ static __be32 nfsd4_encode_components_esc(struct xdr_stream *xdr, char sep, strlen = end - str; if (strlen) { - p = xdr_reserve_space(xdr, strlen + 4); - if (!p) + if (xdr_stream_encode_opaque(xdr, str, strlen) < 0) return nfserr_resource; - p = xdr_encode_opaque(p, str, strlen); count++; - } - else + } else end++; if (found_esc) end = next; @@ -2699,7 +2696,6 @@ static __be32 nfsd4_encode_pathname4(struct xdr_stream *xdr, const struct path *path) { struct path cur = *path; - __be32 *p; struct dentry **components = NULL; unsigned int ncomponents = 0; __be32 err = nfserr_jukebox; @@ -2730,24 +2726,19 @@ static __be32 nfsd4_encode_pathname4(struct xdr_stream *xdr, components[ncomponents++] = cur.dentry; cur.dentry = dget_parent(cur.dentry); } + err = nfserr_resource; - p = xdr_reserve_space(xdr, 4); - if (!p) + if (xdr_stream_encode_u32(xdr, ncomponents) != XDR_UNIT) goto out_free; - *p++ = cpu_to_be32(ncomponents); - while (ncomponents) { struct dentry *dentry = components[ncomponents - 1]; - unsigned int len; spin_lock(&dentry->d_lock); - len = dentry->d_name.len; - p = xdr_reserve_space(xdr, len + 4); - if (!p) { + if (xdr_stream_encode_opaque(xdr, dentry->d_name.name, + dentry->d_name.len) < 0) { spin_unlock(&dentry->d_lock); goto out_free; } - p = xdr_encode_opaque(p, dentry->d_name.name, len); dprintk("/%pd", dentry); spin_unlock(&dentry->d_lock); dput(dentry); @@ -2928,7 +2919,6 @@ struct nfsd4_fattr_args { struct kstat stat; struct kstatfs statfs; struct nfs4_acl *acl; - u64 size; #ifdef CONFIG_NFSD_V4_SECURITY_LABEL void *context; int contextlen; @@ -3040,14 +3030,14 @@ static __be32 nfsd4_encode_fattr4_change(struct xdr_stream *xdr, return nfs_ok; } - c = nfsd4_change_attribute(&args->stat, d_inode(args->dentry)); + c = nfsd4_change_attribute(&args->stat); return nfsd4_encode_changeid4(xdr, c); } static __be32 nfsd4_encode_fattr4_size(struct xdr_stream *xdr, const struct nfsd4_fattr_args *args) { - return nfsd4_encode_uint64_t(xdr, args->size); + return nfsd4_encode_uint64_t(xdr, args->stat.size); } static __be32 nfsd4_encode_fattr4_fsid(struct xdr_stream *xdr, @@ -3512,6 +3502,7 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, int ignore_crossmnt) { DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); + struct nfs4_delegation *dp = NULL; struct nfsd4_fattr_args args; struct svc_fh *tempfh = NULL; int starting_len = xdr->buf->len; @@ -3526,8 +3517,6 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, .dentry = dentry, }; unsigned long bit; - bool file_modified = false; - u64 size = 0; WARN_ON_ONCE(bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1); WARN_ON_ONCE(!nfsd_attrs_supported(minorversion, bmval)); @@ -3555,10 +3544,8 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, if (status) goto out; } - args.size = 0; if (attrmask[0] & (FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE)) { - status = nfsd4_deleg_getattr_conflict(rqstp, dentry, - &file_modified, &size); + status = nfsd4_deleg_getattr_conflict(rqstp, dentry, &dp); if (status) goto out; } @@ -3566,12 +3553,16 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, err = vfs_getattr(&path, &args.stat, STATX_BASIC_STATS | STATX_BTIME | STATX_CHANGE_COOKIE, AT_STATX_SYNC_AS_STAT); + if (dp) { + struct nfs4_cb_fattr *ncf = &dp->dl_cb_fattr; + + if (ncf->ncf_file_modified) + args.stat.size = ncf->ncf_cur_fsize; + + nfs4_put_stid(&dp->dl_stid); + } if (err) goto out_nfserr; - if (file_modified) - args.size = size; - else - args.size = args.stat.size; if (!(args.stat.result_mask & STATX_BTIME)) /* underlying FS does not offer btime so we can't share it */ @@ -3767,7 +3758,7 @@ nfsd4_encode_entry4_fattr(struct nfsd4_readdir *cd, const char *name, nfserr = nfserrno(err); goto out_put; } - nfserr = check_nfsd_access(exp, cd->rd_rqstp); + nfserr = check_nfsd_access(exp, cd->rd_rqstp, false); if (nfserr) goto out_put; @@ -4826,6 +4817,25 @@ nfsd4_encode_server_owner4(struct xdr_stream *xdr, struct svc_rqst *rqstp) } static __be32 +nfsd4_encode_nfs_impl_id4(struct xdr_stream *xdr, struct nfsd4_exchange_id *exid) +{ + __be32 status; + + /* nii_domain */ + status = nfsd4_encode_opaque(xdr, exid->nii_domain.data, + exid->nii_domain.len); + if (status != nfs_ok) + return status; + /* nii_name */ + status = nfsd4_encode_opaque(xdr, exid->nii_name.data, + exid->nii_name.len); + if (status != nfs_ok) + return status; + /* nii_time */ + return nfsd4_encode_nfstime4(xdr, &exid->nii_time); +} + +static __be32 nfsd4_encode_exchange_id(struct nfsd4_compoundres *resp, __be32 nfserr, union nfsd4_op_u *u) { @@ -4859,8 +4869,11 @@ nfsd4_encode_exchange_id(struct nfsd4_compoundres *resp, __be32 nfserr, if (nfserr != nfs_ok) return nfserr; /* eir_server_impl_id<1> */ - if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + if (xdr_stream_encode_u32(xdr, 1) != XDR_UNIT) return nfserr_resource; + nfserr = nfsd4_encode_nfs_impl_id4(xdr, exid); + if (nfserr != nfs_ok) + return nfserr; return nfs_ok; } diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 40ad58a6a036..6a831cb242df 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -320,6 +320,7 @@ __fh_verify(struct svc_rqst *rqstp, { struct nfsd_net *nn = net_generic(net, nfsd_net_id); struct svc_export *exp = NULL; + bool may_bypass_gss = false; struct dentry *dentry; __be32 error; @@ -362,13 +363,12 @@ __fh_verify(struct svc_rqst *rqstp, if (error) goto out; - /* - * pseudoflavor restrictions are not enforced on NLM, - * which clients virtually always use auth_sys for, - * even while using RPCSEC_GSS for NFS. - */ - if (access & NFSD_MAY_LOCK || access & NFSD_MAY_BYPASS_GSS) - goto skip_pseudoflavor_check; + if ((access & NFSD_MAY_NLM) && (exp->ex_flags & NFSEXP_NOAUTHNLM)) + /* NLM is allowed to fully bypass authentication */ + goto out; + + if (access & NFSD_MAY_BYPASS_GSS) + may_bypass_gss = true; /* * Clients may expect to be able to use auth_sys during mount, * even if they use gss for everything else; see section 2.3.2 @@ -376,13 +376,12 @@ __fh_verify(struct svc_rqst *rqstp, */ if (access & NFSD_MAY_BYPASS_GSS_ON_ROOT && exp->ex_path.dentry == dentry) - goto skip_pseudoflavor_check; + may_bypass_gss = true; - error = check_nfsd_access(exp, rqstp); + error = check_nfsd_access(exp, rqstp, may_bypass_gss); if (error) goto out; -skip_pseudoflavor_check: /* Finally, check access permissions. */ error = nfsd_permission(cred, exp, dentry, access); out: @@ -667,20 +666,18 @@ out_negative: __be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp) { bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE); - struct inode *inode; struct kstat stat; __be32 err; if (fhp->fh_no_wcc || fhp->fh_pre_saved) return nfs_ok; - inode = d_inode(fhp->fh_dentry); err = fh_getattr(fhp, &stat); if (err) return err; if (v4) - fhp->fh_pre_change = nfsd4_change_attribute(&stat, inode); + fhp->fh_pre_change = nfsd4_change_attribute(&stat); fhp->fh_pre_mtime = stat.mtime; fhp->fh_pre_ctime = stat.ctime; @@ -697,7 +694,6 @@ __be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp) __be32 fh_fill_post_attrs(struct svc_fh *fhp) { bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE); - struct inode *inode = d_inode(fhp->fh_dentry); __be32 err; if (fhp->fh_no_wcc) @@ -713,7 +709,7 @@ __be32 fh_fill_post_attrs(struct svc_fh *fhp) fhp->fh_post_saved = true; if (v4) fhp->fh_post_change = - nfsd4_change_attribute(&fhp->fh_post_attr, inode); + nfsd4_change_attribute(&fhp->fh_post_attr); return nfs_ok; } @@ -770,7 +766,7 @@ char * SVCFH_fmt(struct svc_fh *fhp) struct knfsd_fh *fh = &fhp->fh_handle; static char buf[2+1+1+64*3+1]; - if (fh->fh_size < 0 || fh->fh_size> 64) + if (fh->fh_size > 64) return "bad-fh"; sprintf(buf, "%d: %*ph", fh->fh_size, fh->fh_size, fh->fh_raw); return buf; @@ -804,7 +800,14 @@ enum fsid_source fsid_source(const struct svc_fh *fhp) return FSIDSOURCE_DEV; } -/* +/** + * nfsd4_change_attribute - Generate an NFSv4 change_attribute value + * @stat: inode attributes + * + * Caller must fill in @stat before calling, typically by invoking + * vfs_getattr() with STATX_MODE, STATX_CTIME, and STATX_CHANGE_COOKIE. + * Returns an unsigned 64-bit changeid4 value (RFC 8881 Section 3.2). + * * We could use i_version alone as the change attribute. However, i_version * can go backwards on a regular file after an unclean shutdown. On its own * that doesn't necessarily cause a problem, but if i_version goes backwards @@ -821,13 +824,13 @@ enum fsid_source fsid_source(const struct svc_fh *fhp) * assume that the new change attr is always logged to stable storage in some * fashion before the results can be seen. */ -u64 nfsd4_change_attribute(const struct kstat *stat, const struct inode *inode) +u64 nfsd4_change_attribute(const struct kstat *stat) { u64 chattr; if (stat->result_mask & STATX_CHANGE_COOKIE) { chattr = stat->change_cookie; - if (S_ISREG(inode->i_mode) && + if (S_ISREG(stat->mode) && !(stat->attributes & STATX_ATTR_CHANGE_MONOTONIC)) { chattr += (u64)stat->ctime.tv_sec << 30; chattr += stat->ctime.tv_nsec; diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h index 5b7394801dc4..876152a91f12 100644 --- a/fs/nfsd/nfsfh.h +++ b/fs/nfsd/nfsfh.h @@ -297,8 +297,7 @@ static inline void fh_clear_pre_post_attrs(struct svc_fh *fhp) fhp->fh_pre_saved = false; } -u64 nfsd4_change_attribute(const struct kstat *stat, - const struct inode *inode); +u64 nfsd4_change_attribute(const struct kstat *stat); __be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp); __be32 fh_fill_post_attrs(struct svc_fh *fhp); __be32 __must_check fh_fill_both_attrs(struct svc_fh *fhp); diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 35b3564c065f..e16bb3717fb9 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -71,8 +71,8 @@ struct nfsd4_callback { struct work_struct cb_work; int cb_seq_status; int cb_status; + int cb_held_slot; bool cb_need_restart; - bool cb_holds_slot; }; struct nfsd4_callback_ops { @@ -137,10 +137,24 @@ struct nfs4_cpntf_state { time64_t cpntf_time; /* last time stateid used */ }; +/* + * RFC 7862 Section 4.8 states: + * + * | A copy offload stateid will be valid until either (A) the client + * | or server restarts or (B) the client returns the resource by + * | issuing an OFFLOAD_CANCEL operation or the client replies to a + * | CB_OFFLOAD operation. + * + * Because a client might not reply to a CB_OFFLOAD, or a reply + * might get lost due to connection loss, NFSD purges async copy + * state after a short period to prevent it from accumulating + * over time. + */ +#define NFSD_COPY_INITIAL_TTL 10 + struct nfs4_cb_fattr { struct nfsd4_callback ncf_getattr; u32 ncf_cb_status; - u32 ncf_cb_bmap[1]; /* from CB_GETATTR reply */ u64 ncf_cb_change; @@ -290,6 +304,9 @@ struct nfsd4_conn { unsigned char cn_flags; }; +/* Maximum number of slots that nfsd will use in the backchannel */ +#define NFSD_BC_SLOT_TABLE_SIZE (sizeof(u32) * 8) + /* * Representation of a v4.1+ session. These are refcounted in a similar fashion * to the nfs4_client. References are only taken when the server is actively @@ -297,19 +314,19 @@ struct nfsd4_conn { */ struct nfsd4_session { atomic_t se_ref; + spinlock_t se_lock; + u32 se_cb_slot_avail; /* bitmap of available slots */ + u32 se_cb_highest_slot; /* highest slot client wants */ + u32 se_cb_prog; + bool se_dead; struct list_head se_hash; /* hash by sessionid */ struct list_head se_perclnt; -/* See SESSION4_PERSIST, etc. for standard flags; this is internal-only: */ -#define NFS4_SESSION_DEAD 0x010 - u32 se_flags; struct nfs4_client *se_client; struct nfs4_sessionid se_sessionid; struct nfsd4_channel_attrs se_fchannel; - struct nfsd4_channel_attrs se_bchannel; struct nfsd4_cb_sec se_cb_sec; struct list_head se_conns; - u32 se_cb_prog; - u32 se_cb_seq_nr; + u32 se_cb_seq_nr[NFSD_BC_SLOT_TABLE_SIZE]; struct nfsd4_slot *se_slots[]; /* forward channel slots */ }; @@ -443,9 +460,6 @@ struct nfs4_client { */ struct dentry *cl_nfsd_info_dentry; - /* for nfs41 callbacks */ - /* We currently support a single back channel with a single slot */ - unsigned long cl_cb_slot_busy; struct rpc_wait_queue cl_cb_waitq; /* backchannel callers may */ /* wait here for slots */ struct net *net; @@ -742,6 +756,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, extern bool nfsd4_run_cb(struct nfsd4_callback *cb); extern void nfsd4_shutdown_callback(struct nfs4_client *); extern void nfsd4_shutdown_copy(struct nfs4_client *clp); +void nfsd4_async_copy_reaper(struct nfsd_net *nn); +bool nfsd4_has_active_async_copies(struct nfs4_client *clp); extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, struct nfsd_net *nn); extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn); @@ -784,5 +800,5 @@ static inline bool try_to_expire_client(struct nfs4_client *clp) } extern __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, - struct dentry *dentry, bool *file_modified, u64 *size); + struct dentry *dentry, struct nfs4_delegation **pdp); #endif /* NFSD4_STATE_H */ diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index b8470d4cbe99..696c89f68a9e 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -79,7 +79,7 @@ DEFINE_NFSD_XDR_ERR_EVENT(cant_encode); { NFSD_MAY_READ, "READ" }, \ { NFSD_MAY_SATTR, "SATTR" }, \ { NFSD_MAY_TRUNC, "TRUNC" }, \ - { NFSD_MAY_LOCK, "LOCK" }, \ + { NFSD_MAY_NLM, "NLM" }, \ { NFSD_MAY_OWNER_OVERRIDE, "OWNER_OVERRIDE" }, \ { NFSD_MAY_LOCAL_ACCESS, "LOCAL_ACCESS" }, \ { NFSD_MAY_BYPASS_GSS_ON_ROOT, "BYPASS_GSS_ON_ROOT" }, \ @@ -163,7 +163,7 @@ TRACE_EVENT(nfsd_compound_decode_err, __entry->opnum, __entry->status) ); -TRACE_EVENT(nfsd_compound_encode_err, +DECLARE_EVENT_CLASS(nfsd_compound_err_class, TP_PROTO( const struct svc_rqst *rqstp, u32 opnum, @@ -184,6 +184,18 @@ TRACE_EVENT(nfsd_compound_encode_err, __entry->opnum, __entry->status) ); +#define DEFINE_NFSD_COMPOUND_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_compound_err_class, nfsd_compound_##name##_err, \ + TP_PROTO( \ + const struct svc_rqst *rqstp, \ + u32 opnum, \ + __be32 status \ + ), \ + TP_ARGS(rqstp, opnum, status)) + +DEFINE_NFSD_COMPOUND_ERR_EVENT(op); +DEFINE_NFSD_COMPOUND_ERR_EVENT(encode); + #define show_fs_file_type(x) \ __print_symbolic(x, \ { S_IFLNK, "LNK" }, \ @@ -1685,7 +1697,7 @@ TRACE_EVENT(nfsd_cb_free_slot, __entry->cl_id = sid->clientid.cl_id; __entry->seqno = sid->sequence; __entry->reserved = sid->reserved; - __entry->slot_seqno = session->se_cb_seq_nr; + __entry->slot_seqno = session->se_cb_seq_nr[cb->cb_held_slot]; ), TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " sessionid=%08x:%08x:%08x:%08x new slot seqno=%u", @@ -2232,7 +2244,7 @@ TRACE_EVENT(nfsd_copy_done, ) ); -TRACE_EVENT(nfsd_copy_async_done, +DECLARE_EVENT_CLASS(nfsd_copy_async_done_class, TP_PROTO( const struct nfsd4_copy *copy ), @@ -2301,6 +2313,15 @@ TRACE_EVENT(nfsd_copy_async_done, ) ); +#define DEFINE_COPY_ASYNC_DONE_EVENT(name) \ +DEFINE_EVENT(nfsd_copy_async_done_class, \ + nfsd_copy_async_##name, \ + TP_PROTO(const struct nfsd4_copy *copy), \ + TP_ARGS(copy)) + +DEFINE_COPY_ASYNC_DONE_EVENT(done); +DEFINE_COPY_ASYNC_DONE_EVENT(cancel); + #endif /* _NFSD_TRACE_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index d6d4f2a0e898..29cb7b812d71 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -35,7 +35,6 @@ #include "xdr3.h" #ifdef CONFIG_NFSD_V4 -#include "../internal.h" #include "acl.h" #include "idmap.h" #include "xdr4.h" @@ -321,7 +320,7 @@ nfsd_lookup(struct svc_rqst *rqstp, struct svc_fh *fhp, const char *name, err = nfsd_lookup_dentry(rqstp, fhp, name, len, &exp, &dentry); if (err) return err; - err = check_nfsd_access(exp, rqstp); + err = check_nfsd_access(exp, rqstp, false); if (err) goto out; /* @@ -861,8 +860,7 @@ int nfsd_open_break_lease(struct inode *inode, int access) * N.B. After this call fhp needs an fh_put */ static int -__nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, - int may_flags, struct file **filp) +__nfsd_open(struct svc_fh *fhp, umode_t type, int may_flags, struct file **filp) { struct path path; struct inode *inode; @@ -932,7 +930,7 @@ nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, retry: err = fh_verify(rqstp, fhp, type, may_flags); if (!err) { - host_err = __nfsd_open(rqstp, fhp, type, may_flags, filp); + host_err = __nfsd_open(fhp, type, may_flags, filp); if (host_err == -EOPENSTALE && !retried) { retried = true; fh_put(fhp); @@ -945,7 +943,6 @@ retry: /** * nfsd_open_verified - Open a regular file for the filecache - * @rqstp: RPC request * @fhp: NFS filehandle of the file to open * @may_flags: internal permission flags * @filp: OUT: open "struct file *" @@ -953,10 +950,9 @@ retry: * Returns zero on success, or a negative errno value. */ int -nfsd_open_verified(struct svc_rqst *rqstp, struct svc_fh *fhp, int may_flags, - struct file **filp) +nfsd_open_verified(struct svc_fh *fhp, int may_flags, struct file **filp) { - return __nfsd_open(rqstp, fhp, S_IFREG, may_flags, filp); + return __nfsd_open(fhp, S_IFREG, may_flags, filp); } /* @@ -2509,7 +2505,7 @@ nfsd_permission(struct svc_cred *cred, struct svc_export *exp, (acc & NFSD_MAY_EXEC)? " exec" : "", (acc & NFSD_MAY_SATTR)? " sattr" : "", (acc & NFSD_MAY_TRUNC)? " trunc" : "", - (acc & NFSD_MAY_LOCK)? " lock" : "", + (acc & NFSD_MAY_NLM)? " nlm" : "", (acc & NFSD_MAY_OWNER_OVERRIDE)? " owneroverride" : "", inode->i_mode, IS_IMMUTABLE(inode)? " immut" : "", @@ -2534,16 +2530,6 @@ nfsd_permission(struct svc_cred *cred, struct svc_export *exp, if ((acc & NFSD_MAY_TRUNC) && IS_APPEND(inode)) return nfserr_perm; - if (acc & NFSD_MAY_LOCK) { - /* If we cannot rely on authentication in NLM requests, - * just allow locks, otherwise require read permission, or - * ownership - */ - if (exp->ex_flags & NFSEXP_NOAUTHNLM) - return 0; - else - acc = NFSD_MAY_READ | NFSD_MAY_OWNER_OVERRIDE; - } /* * The file owner always gets access permission for accesses that * would normally be checked at open time. This is to make diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 3ff146522556..f9b09b842856 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -20,7 +20,7 @@ #define NFSD_MAY_READ 0x004 /* == MAY_READ */ #define NFSD_MAY_SATTR 0x008 #define NFSD_MAY_TRUNC 0x010 -#define NFSD_MAY_LOCK 0x020 +#define NFSD_MAY_NLM 0x020 /* request is from lockd */ #define NFSD_MAY_MASK 0x03f /* extra hints to permission and open routines: */ @@ -114,8 +114,8 @@ __be32 nfsd_setxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, int nfsd_open_break_lease(struct inode *, int); __be32 nfsd_open(struct svc_rqst *, struct svc_fh *, umode_t, int, struct file **); -int nfsd_open_verified(struct svc_rqst *rqstp, struct svc_fh *fhp, - int may_flags, struct file **filp); +int nfsd_open_verified(struct svc_fh *fhp, int may_flags, + struct file **filp); __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, struct file *file, loff_t offset, unsigned long *count, diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 2a21a7662e03..382cc1389396 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -567,6 +567,7 @@ struct nfsd4_exchange_id { struct xdr_netobj nii_domain; struct xdr_netobj nii_name; struct timespec64 nii_time; + char *server_impl_name; }; struct nfsd4_sequence { @@ -675,6 +676,7 @@ struct nfsd4_cb_offload { struct nfsd4_callback co_cb; struct nfsd42_write_res co_res; __be32 co_nfserr; + unsigned int co_retries; struct knfsd_fh co_fh; }; @@ -693,12 +695,16 @@ struct nfsd4_copy { #define NFSD4_COPY_F_SYNCHRONOUS (2) #define NFSD4_COPY_F_COMMITTED (3) #define NFSD4_COPY_F_COMPLETED (4) +#define NFSD4_COPY_F_OFFLOAD_DONE (5) /* response */ __be32 nfserr; struct nfsd42_write_res cp_res; struct knfsd_fh fh; + /* offload callback */ + struct nfsd4_cb_offload cp_cb_offload; + struct nfs4_client *cp_clp; struct nfsd_file *nf_src; @@ -709,6 +715,7 @@ struct nfsd4_copy { struct list_head copies; struct task_struct *copy_task; refcount_t refcount; + unsigned int cp_ttl; struct nfsd4_ssc_umount_item *ss_nsui; struct nfs_fh c_fh; @@ -930,6 +937,7 @@ extern __be32 nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *, union nfsd4_op_u *u); extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp, struct nfsd4_compound_state *, union nfsd4_op_u *u); +void nfsd4_exchange_id_release(union nfsd4_op_u *u); extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp, struct nfsd4_compound_state *, union nfsd4_op_u *u); extern __be32 nfsd4_backchannel_ctl(struct svc_rqst *, diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 1ab165c2939f..4cc8801e50e3 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -160,6 +160,19 @@ struct fid { #define EXPORT_FH_FID 0x2 /* File handle may be non-decodeable */ #define EXPORT_FH_DIR_ONLY 0x4 /* Only decode file handle for a directory */ +/* + * Filesystems use only lower 8 bits of file_handle type for fid_type. + * name_to_handle_at() uses upper 16 bits of type as user flags to be + * interpreted by open_by_handle_at(). + */ +#define FILEID_USER_FLAGS_MASK 0xffff0000 +#define FILEID_USER_FLAGS(type) ((type) & FILEID_USER_FLAGS_MASK) + +/* Flags supported in encoded handle_type that is exported to user */ +#define FILEID_IS_CONNECTABLE 0x10000 +#define FILEID_IS_DIR 0x20000 +#define FILEID_VALID_USER_FLAGS (FILEID_IS_CONNECTABLE | FILEID_IS_DIR) + /** * struct export_operations - for nfsd to communicate with file systems * @encode_fh: encode a file handle fragment from a dentry diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index b0b821edfd97..c24f8bc01045 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -24,10 +24,11 @@ #define NEW_ADDR ((block_t)-1) /* used as block_t addresses */ #define COMPRESS_ADDR ((block_t)-2) /* used as compressed data flag */ -#define F2FS_BYTES_TO_BLK(bytes) ((bytes) >> F2FS_BLKSIZE_BITS) -#define F2FS_BLK_TO_BYTES(blk) ((blk) << F2FS_BLKSIZE_BITS) +#define F2FS_BLKSIZE_MASK (F2FS_BLKSIZE - 1) +#define F2FS_BYTES_TO_BLK(bytes) ((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS) +#define F2FS_BLK_TO_BYTES(blk) ((unsigned long long)(blk) << F2FS_BLKSIZE_BITS) #define F2FS_BLK_END_BYTES(blk) (F2FS_BLK_TO_BYTES(blk + 1) - 1) -#define F2FS_BLK_ALIGN(x) (F2FS_BYTES_TO_BLK((x) + F2FS_BLKSIZE - 1)) +#define F2FS_BLK_ALIGN(x) (F2FS_BYTES_TO_BLK((x) + F2FS_BLKSIZE - 1)) /* 0, 1(node nid), 2(meta nid) are reserved node id */ #define F2FS_RESERVED_NODE_NUM 3 diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 2a1ed05d5782..12d532b012c5 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -298,7 +298,9 @@ enum i3c_open_drain_speed { * @I3C_ADDR_SLOT_I2C_DEV: address is assigned to an I2C device * @I3C_ADDR_SLOT_I3C_DEV: address is assigned to an I3C device * @I3C_ADDR_SLOT_STATUS_MASK: address slot mask - * + * @I3C_ADDR_SLOT_EXT_STATUS_MASK: address slot mask with extended information + * @I3C_ADDR_SLOT_EXT_DESIRED: the bitmask represents addresses that are preferred by some devices, + * such as the "assigned-address" property in a device tree source. * On an I3C bus, addresses are assigned dynamically, and we need to know which * addresses are free to use and which ones are already assigned. * @@ -311,8 +313,12 @@ enum i3c_addr_slot_status { I3C_ADDR_SLOT_I2C_DEV, I3C_ADDR_SLOT_I3C_DEV, I3C_ADDR_SLOT_STATUS_MASK = 3, + I3C_ADDR_SLOT_EXT_STATUS_MASK = 7, + I3C_ADDR_SLOT_EXT_DESIRED = BIT(2), }; +#define I3C_ADDR_SLOT_STATUS_BITS 4 + /** * struct i3c_bus - I3C bus object * @cur_master: I3C master currently driving the bus. Since I3C is multi-master @@ -354,7 +360,7 @@ enum i3c_addr_slot_status { struct i3c_bus { struct i3c_dev_desc *cur_master; int id; - unsigned long addrslots[((I2C_MAX_ADDR + 1) * 2) / BITS_PER_LONG]; + unsigned long addrslots[((I2C_MAX_ADDR + 1) * I3C_ADDR_SLOT_STATUS_BITS) / BITS_PER_LONG]; enum i3c_bus_mode mode; struct { unsigned long i3c; diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 6e9fb667a1c5..5385349f0b8a 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -249,6 +249,38 @@ struct resource *lookup_resource(struct resource *root, resource_size_t start); int adjust_resource(struct resource *res, resource_size_t start, resource_size_t size); resource_size_t resource_alignment(struct resource *res); + +/** + * resource_set_size - Calculate resource end address from size and start + * @res: Resource descriptor + * @size: Size of the resource + * + * Calculate the end address for @res based on @size. + * + * Note: The start address of @res must be set when calling this function. + * Prefer resource_set_range() if setting both the start address and @size. + */ +static inline void resource_set_size(struct resource *res, resource_size_t size) +{ + res->end = res->start + size - 1; +} + +/** + * resource_set_range - Set resource start and end addresses + * @res: Resource descriptor + * @start: Start address for the resource + * @size: Size of the resource + * + * Set @res start address and calculate the end address based on @size. + */ +static inline void resource_set_range(struct resource *res, + resource_size_t start, + resource_size_t size) +{ + res->start = start; + resource_set_size(res, size); +} + static inline resource_size_t resource_size(const struct resource *res) { return res->end - res->start + 1; diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index 61c4b9c41904..c8f0f9458f2c 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -278,9 +278,9 @@ __be32 nlmsvc_lock(struct svc_rqst *, struct nlm_file *, struct nlm_host *, struct nlm_lock *, int, struct nlm_cookie *, int); __be32 nlmsvc_unlock(struct net *net, struct nlm_file *, struct nlm_lock *); -__be32 nlmsvc_testlock(struct svc_rqst *, struct nlm_file *, - struct nlm_host *, struct nlm_lock *, - struct nlm_lock *, struct nlm_cookie *); +__be32 nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, + struct nlm_host *host, struct nlm_lock *lock, + struct nlm_lock *conflock); __be32 nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct nlm_lock *); void nlmsvc_retry_blocked(struct svc_rqst *rqstp); void nlmsvc_traverse_blocks(struct nlm_host *, struct nlm_file *, diff --git a/include/linux/lockd/xdr.h b/include/linux/lockd/xdr.h index 80cca9426761..17d53165d9f2 100644 --- a/include/linux/lockd/xdr.h +++ b/include/linux/lockd/xdr.h @@ -73,8 +73,6 @@ struct nlm_args { u32 fsm_mode; }; -typedef struct nlm_args nlm_args; - /* * Generic lockd result */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 2bbf73eb53e7..c39c4945946c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2550,6 +2550,7 @@ struct kvec; struct page *get_dump_page(unsigned long addr); bool folio_mark_dirty(struct folio *folio); +bool folio_mark_dirty_lock(struct folio *folio); bool set_page_dirty(struct page *page); int set_page_dirty_lock(struct page *page); diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 3982fea79919..9202f4b24343 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -55,7 +55,7 @@ struct nfsd_localio_operations { const struct cred *, const struct nfs_fh *, const fmode_t); - void (*nfsd_file_put_local)(struct nfsd_file *); + struct net *(*nfsd_file_put_local)(struct nfsd_file *); struct file *(*nfsd_file_file)(struct nfsd_file *); } ____cacheline_aligned; @@ -66,7 +66,7 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *, struct rpc_clnt *, const struct cred *, const struct nfs_fh *, const fmode_t); -static inline void nfs_to_nfsd_file_put_local(struct nfsd_file *localio) +static inline void nfs_to_nfsd_net_put(struct net *net) { /* * Once reference to nfsd_serv is dropped, NFSD could be @@ -74,10 +74,22 @@ static inline void nfs_to_nfsd_file_put_local(struct nfsd_file *localio) * by always taking RCU. */ rcu_read_lock(); - nfs_to->nfsd_file_put_local(localio); + nfs_to->nfsd_serv_put(net); rcu_read_unlock(); } +static inline void nfs_to_nfsd_file_put_local(struct nfsd_file *localio) +{ + /* + * Must not hold RCU otherwise nfsd_file_put() can easily trigger: + * "Voluntary context switch within RCU read-side critical section!" + * by scheduling deep in underlying filesystem (e.g. XFS). + */ + struct net *net = nfs_to->nfsd_file_put_local(localio); + + nfs_to_nfsd_net_put(net); +} + #else /* CONFIG_NFS_LOCALIO */ static inline void nfsd_localio_ops_init(void) { diff --git a/include/linux/pci-bwctrl.h b/include/linux/pci-bwctrl.h new file mode 100644 index 000000000000..cee07127455b --- /dev/null +++ b/include/linux/pci-bwctrl.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * PCIe bandwidth controller + * + * Copyright (C) 2023-2024 Intel Corporation + */ + +#ifndef LINUX_PCI_BWCTRL_H +#define LINUX_PCI_BWCTRL_H + +#include <linux/pci.h> + +struct thermal_cooling_device; + +#ifdef CONFIG_PCIE_THERMAL +struct thermal_cooling_device *pcie_cooling_device_register(struct pci_dev *port); +void pcie_cooling_device_unregister(struct thermal_cooling_device *cdev); +#else +static inline struct thermal_cooling_device *pcie_cooling_device_register(struct pci_dev *port) +{ + return NULL; +} +static inline void pcie_cooling_device_unregister(struct thermal_cooling_device *cdev) +{ +} +#endif + +#endif diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 42ef06136bd1..e818e3fdcded 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -33,10 +33,42 @@ pci_epc_interface_string(enum pci_epc_interface_type type) } /** + * struct pci_epc_map - information about EPC memory for mapping a RC PCI + * address range + * @pci_addr: start address of the RC PCI address range to map + * @pci_size: size of the RC PCI address range mapped from @pci_addr + * @map_pci_addr: RC PCI address used as the first address mapped (may be lower + * than @pci_addr) + * @map_size: size of the controller memory needed for mapping the RC PCI address + * range @map_pci_addr..@pci_addr+@pci_size + * @phys_base: base physical address of the allocated EPC memory for mapping the + * RC PCI address range + * @phys_addr: physical address at which @pci_addr is mapped + * @virt_base: base virtual address of the allocated EPC memory for mapping the + * RC PCI address range + * @virt_addr: virtual address at which @pci_addr is mapped + */ +struct pci_epc_map { + u64 pci_addr; + size_t pci_size; + + u64 map_pci_addr; + size_t map_size; + + phys_addr_t phys_base; + phys_addr_t phys_addr; + void __iomem *virt_base; + void __iomem *virt_addr; +}; + +/** * struct pci_epc_ops - set of function pointers for performing EPC operations * @write_header: ops to populate configuration space header * @set_bar: ops to configure the BAR * @clear_bar: ops to reset the BAR + * @align_addr: operation to get the mapping address, mapping size and offset + * into a controller memory window needed to map an RC PCI address + * region * @map_addr: ops to map CPU address to PCI address * @unmap_addr: ops to unmap CPU address and PCI address * @set_msi: ops to set the requested number of MSI interrupts in the MSI @@ -61,6 +93,8 @@ struct pci_epc_ops { struct pci_epf_bar *epf_bar); void (*clear_bar)(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar); + u64 (*align_addr)(struct pci_epc *epc, u64 pci_addr, size_t *size, + size_t *offset); int (*map_addr)(struct pci_epc *epc, u8 func_no, u8 vfunc_no, phys_addr_t addr, u64 pci_addr, size_t size); void (*unmap_addr)(struct pci_epc *epc, u8 func_no, u8 vfunc_no, @@ -278,6 +312,10 @@ void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc, phys_addr_t *phys_addr, size_t size); void pci_epc_mem_free_addr(struct pci_epc *epc, phys_addr_t phys_addr, void __iomem *virt_addr, size_t size); +int pci_epc_mem_map(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u64 pci_addr, size_t pci_size, struct pci_epc_map *map); +void pci_epc_mem_unmap(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_map *map); #else static inline void pci_epc_init_notify(struct pci_epc *epc) diff --git a/include/linux/pci-pwrctl.h b/include/linux/pci-pwrctrl.h index 0d23dddf59ec..7d439b0675e9 100644 --- a/include/linux/pci-pwrctl.h +++ b/include/linux/pci-pwrctrl.h @@ -3,8 +3,8 @@ * Copyright (C) 2024 Linaro Ltd. */ -#ifndef __PCI_PWRCTL_H__ -#define __PCI_PWRCTL_H__ +#ifndef __PCI_PWRCTRL_H__ +#define __PCI_PWRCTRL_H__ #include <linux/notifier.h> #include <linux/workqueue.h> @@ -29,14 +29,14 @@ struct device_link; */ /** - * struct pci_pwrctl - PCI device power control context. + * struct pci_pwrctrl - PCI device power control context. * @dev: Address of the power controlling device. * * An object of this type must be allocated by the PCI power control device and - * passed to the pwrctl subsystem to trigger a bus rescan and setup a device + * passed to the pwrctrl subsystem to trigger a bus rescan and setup a device * link with the device once it's up. */ -struct pci_pwrctl { +struct pci_pwrctrl { struct device *dev; /* Private: don't use. */ @@ -45,10 +45,10 @@ struct pci_pwrctl { struct work_struct work; }; -void pci_pwrctl_init(struct pci_pwrctl *pwrctl, struct device *dev); -int pci_pwrctl_device_set_ready(struct pci_pwrctl *pwrctl); -void pci_pwrctl_device_unset_ready(struct pci_pwrctl *pwrctl); -int devm_pci_pwrctl_device_set_ready(struct device *dev, - struct pci_pwrctl *pwrctl); +void pci_pwrctrl_init(struct pci_pwrctrl *pwrctrl, struct device *dev); +int pci_pwrctrl_device_set_ready(struct pci_pwrctrl *pwrctrl); +void pci_pwrctrl_device_unset_ready(struct pci_pwrctrl *pwrctrl); +int devm_pci_pwrctrl_device_set_ready(struct device *dev, + struct pci_pwrctrl *pwrctrl); -#endif /* __PCI_PWRCTL_H__ */ +#endif /* __PCI_PWRCTRL_H__ */ diff --git a/include/linux/pci-tph.h b/include/linux/pci-tph.h new file mode 100644 index 000000000000..c3e806c13d64 --- /dev/null +++ b/include/linux/pci-tph.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * TPH (TLP Processing Hints) + * + * Copyright (C) 2024 Advanced Micro Devices, Inc. + * Eric Van Tassell <Eric.VanTassell@amd.com> + * Wei Huang <wei.huang2@amd.com> + */ +#ifndef LINUX_PCI_TPH_H +#define LINUX_PCI_TPH_H + +/* + * According to the ECN for PCI Firmware Spec, Steering Tag can be different + * depending on the memory type: Volatile Memory or Persistent Memory. When a + * caller query about a target's Steering Tag, it must provide the target's + * tph_mem_type. ECN link: https://members.pcisig.com/wg/PCI-SIG/document/15470. + */ +enum tph_mem_type { + TPH_MEM_TYPE_VM, /* volatile memory */ + TPH_MEM_TYPE_PM /* persistent memory */ +}; + +#ifdef CONFIG_PCIE_TPH +int pcie_tph_set_st_entry(struct pci_dev *pdev, + unsigned int index, u16 tag); +int pcie_tph_get_cpu_st(struct pci_dev *dev, + enum tph_mem_type mem_type, + unsigned int cpu_uid, u16 *tag); +void pcie_disable_tph(struct pci_dev *pdev); +int pcie_enable_tph(struct pci_dev *pdev, int mode); +#else +static inline int pcie_tph_set_st_entry(struct pci_dev *pdev, + unsigned int index, u16 tag) +{ return -EINVAL; } +static inline int pcie_tph_get_cpu_st(struct pci_dev *dev, + enum tph_mem_type mem_type, + unsigned int cpu_uid, u16 *tag) +{ return -EINVAL; } +static inline void pcie_disable_tph(struct pci_dev *pdev) { } +static inline int pcie_enable_tph(struct pci_dev *pdev, int mode) +{ return -EINVAL; } +#endif + +#endif /* LINUX_PCI_TPH_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 733ff6570e2d..db9b47ce3eef 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -313,12 +313,20 @@ struct pci_vpd { }; struct irq_affinity; +struct pcie_bwctrl_data; struct pcie_link_state; struct pci_sriov; struct pci_p2pdma; struct rcec_ea; -/* The pci_dev structure describes PCI devices */ +/* struct pci_dev - describes a PCI device + * + * @supported_speeds: PCIe Supported Link Speeds Vector (+ reserved 0 at + * LSB). 0 when the supported speeds cannot be + * determined (e.g., for Root Complex Integrated + * Endpoints without the relevant Capability + * Registers). + */ struct pci_dev { struct list_head bus_list; /* Node in per-bus list */ struct pci_bus *bus; /* Bus this device is on */ @@ -434,6 +442,7 @@ struct pci_dev { unsigned int ats_enabled:1; /* Address Translation Svc */ unsigned int pasid_enabled:1; /* Process Address Space ID */ unsigned int pri_enabled:1; /* Page Request Interface */ + unsigned int tph_enabled:1; /* TLP Processing Hints */ unsigned int is_managed:1; /* Managed via devres */ unsigned int is_msi_managed:1; /* MSI release via devres installed */ unsigned int needs_freset:1; /* Requires fundamental reset */ @@ -495,6 +504,7 @@ struct pci_dev { unsigned int dpc_rp_extensions:1; u8 dpc_rp_log_size; #endif + struct pcie_bwctrl_data *link_bwctrl; #ifdef CONFIG_PCI_ATS union { struct pci_sriov *sriov; /* PF: SR-IOV info */ @@ -522,6 +532,7 @@ struct pci_dev { struct npem *npem; /* Native PCIe Enclosure Management */ #endif u16 acs_cap; /* ACS Capability offset */ + u8 supported_speeds; /* Supported Link Speeds Vector */ phys_addr_t rom; /* Physical address if not from BAR */ size_t romlen; /* Length if not from BAR */ /* @@ -534,6 +545,12 @@ struct pci_dev { /* These methods index pci_reset_fn_methods[] */ u8 reset_methods[PCI_NUM_RESET_METHODS]; /* In priority order */ + +#ifdef CONFIG_PCIE_TPH + u16 tph_cap; /* TPH capability offset */ + u8 tph_mode; /* TPH mode */ + u8 tph_req_type; /* TPH requester type */ +#endif }; static inline struct pci_dev *pci_physfn(struct pci_dev *dev) @@ -626,27 +643,6 @@ void pci_set_host_bridge_release(struct pci_host_bridge *bridge, int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge); -/* - * The first PCI_BRIDGE_RESOURCE_NUM PCI bus resources (those that correspond - * to P2P or CardBus bridge windows) go in a table. Additional ones (for - * buses below host bridges or subtractive decode bridges) go in the list. - * Use pci_bus_for_each_resource() to iterate through all the resources. - */ - -/* - * PCI_SUBTRACTIVE_DECODE means the bridge forwards the window implicitly - * and there's no way to program the bridge with the details of the window. - * This does not apply to ACPI _CRS windows, even with the _DEC subtractive- - * decode bit set, because they are explicit and can be programmed with _SRS. - */ -#define PCI_SUBTRACTIVE_DECODE 0x1 - -struct pci_bus_resource { - struct list_head list; - struct resource *res; - unsigned int flags; -}; - #define PCI_REGION_FLAG_MASK 0x0fU /* These bits of resource flags tell us the PCI region flags */ struct pci_bus { @@ -1274,6 +1270,7 @@ static inline int pcie_capability_clear_and_set_word(struct pci_dev *dev, { switch (pos) { case PCI_EXP_LNKCTL: + case PCI_EXP_LNKCTL2: case PCI_EXP_RTCTL: return pcie_capability_clear_and_set_word_locked(dev, pos, clear, set); @@ -1498,8 +1495,7 @@ void pci_add_resource(struct list_head *resources, struct resource *res); void pci_add_resource_offset(struct list_head *resources, struct resource *res, resource_size_t offset); void pci_free_resource_list(struct list_head *resources); -void pci_bus_add_resource(struct pci_bus *bus, struct resource *res, - unsigned int flags); +void pci_bus_add_resource(struct pci_bus *bus, struct resource *res); struct resource *pci_bus_resource_n(const struct pci_bus *bus, int n); void pci_bus_remove_resources(struct pci_bus *bus); void pci_bus_remove_resource(struct pci_bus *bus, struct resource *res); @@ -1623,8 +1619,6 @@ int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, void pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), void *userdata); -void pci_walk_bus_locked(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), - void *userdata); int pci_cfg_space_size(struct pci_dev *dev); unsigned char pci_bus_max_busnr(struct pci_bus *bus); void pci_setup_bridge(struct pci_bus *bus); @@ -1786,9 +1780,19 @@ static inline int pci_irqd_intx_xlate(struct irq_domain *d, #ifdef CONFIG_PCIEPORTBUS extern bool pcie_ports_disabled; extern bool pcie_ports_native; + +int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, + bool use_lt); #else #define pcie_ports_disabled true #define pcie_ports_native false + +static inline int pcie_set_target_speed(struct pci_dev *port, + enum pci_bus_speed speed_req, + bool use_lt) +{ + return -EOPNOTSUPP; +} #endif #define PCIE_LINK_STATE_L0S (BIT(0) | BIT(1)) /* Upstr/dwnstr L0s */ @@ -2293,15 +2297,15 @@ static inline void pci_fixup_device(enum pci_fixup_pass pass, struct pci_dev *dev) { } #endif +int pcim_request_all_regions(struct pci_dev *pdev, const char *name); void __iomem *pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen); void __iomem *pcim_iomap_region(struct pci_dev *pdev, int bar, const char *name); +void pcim_iounmap_region(struct pci_dev *pdev, int bar); void pcim_iounmap(struct pci_dev *pdev, void __iomem *addr); void __iomem * const *pcim_iomap_table(struct pci_dev *pdev); int pcim_request_region(struct pci_dev *pdev, int bar, const char *name); int pcim_iomap_regions(struct pci_dev *pdev, int mask, const char *name); -int pcim_iomap_regions_request_all(struct pci_dev *pdev, int mask, - const char *name); void pcim_iounmap_regions(struct pci_dev *pdev, int mask); void __iomem *pcim_iomap_range(struct pci_dev *pdev, int bar, unsigned long offset, unsigned long len); @@ -2609,6 +2613,12 @@ pci_host_bridge_acpi_msi_domain(struct pci_bus *bus) { return NULL; } static inline bool pci_pr3_present(struct pci_dev *pdev) { return false; } #endif +#if defined(CONFIG_X86) && defined(CONFIG_ACPI) +bool arch_pci_dev_is_removable(struct pci_dev *pdev); +#else +static inline bool arch_pci_dev_is_removable(struct pci_dev *pdev) { return false; } +#endif + #ifdef CONFIG_EEH static inline struct eeh_dev *pci_dev_to_eeh_dev(struct pci_dev *pdev) { diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 5f775e104f9a..a2ab813a9800 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -681,6 +681,27 @@ xdr_stream_decode_u32(struct xdr_stream *xdr, __u32 *ptr) } /** + * xdr_stream_decode_be32 - Decode a big-endian 32-bit integer + * @xdr: pointer to xdr_stream + * @ptr: location to store integer + * + * Return values: + * %0 on success + * %-EBADMSG on XDR buffer overflow + */ +static inline ssize_t +xdr_stream_decode_be32(struct xdr_stream *xdr, __be32 *ptr) +{ + const size_t count = sizeof(*ptr); + __be32 *p = xdr_inline_decode(xdr, count); + + if (unlikely(!p)) + return -EBADMSG; + *ptr = *p; + return 0; +} + +/** * xdr_stream_decode_u64 - Decode a 64-bit integer * @xdr: pointer to xdr_stream * @ptr: location to store 64-bit integer diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h index be9e62371758..20c7270aa64d 100644 --- a/include/linux/sunrpc/xdrgen/_defs.h +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -23,4 +23,13 @@ typedef struct { u8 *data; } opaque; +#define XDR_void (0) +#define XDR_bool (1) +#define XDR_int (1) +#define XDR_unsigned_int (1) +#define XDR_long (1) +#define XDR_unsigned_long (1) +#define XDR_hyper (2) +#define XDR_unsigned_hyper (2) + #endif /* _SUNRPC_XDRGEN__DEFS_H_ */ diff --git a/include/uapi/linux/f2fs.h b/include/uapi/linux/f2fs.h index 955d440be104..f7aaf8d23e20 100644 --- a/include/uapi/linux/f2fs.h +++ b/include/uapi/linux/f2fs.h @@ -43,6 +43,7 @@ #define F2FS_IOC_DECOMPRESS_FILE _IO(F2FS_IOCTL_MAGIC, 23) #define F2FS_IOC_COMPRESS_FILE _IO(F2FS_IOCTL_MAGIC, 24) #define F2FS_IOC_START_ATOMIC_REPLACE _IO(F2FS_IOCTL_MAGIC, 25) +#define F2FS_IOC_GET_DEV_ALIAS_FILE _IOR(F2FS_IOCTL_MAGIC, 26, __u32) /* * should be same as XFS_IOC_GOINGDOWN. diff --git a/include/uapi/linux/fcntl.h b/include/uapi/linux/fcntl.h index a40833bf2855..6e6907e63bfc 100644 --- a/include/uapi/linux/fcntl.h +++ b/include/uapi/linux/fcntl.h @@ -153,5 +153,6 @@ object identity and may not be usable with open_by_handle_at(2). */ #define AT_HANDLE_MNT_ID_UNIQUE 0x001 /* Return the u64 unique mount ID. */ +#define AT_HANDLE_CONNECTABLE 0x002 /* Request a connectable file handle */ #endif /* _UAPI_LINUX_FCNTL_H */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 12323b3334a9..1601c7ed5fab 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -340,7 +340,8 @@ #define PCI_MSIX_ENTRY_UPPER_ADDR 0x4 /* Message Upper Address */ #define PCI_MSIX_ENTRY_DATA 0x8 /* Message Data */ #define PCI_MSIX_ENTRY_VECTOR_CTRL 0xc /* Vector Control */ -#define PCI_MSIX_ENTRY_CTRL_MASKBIT 0x00000001 +#define PCI_MSIX_ENTRY_CTRL_MASKBIT 0x00000001 /* Mask Bit */ +#define PCI_MSIX_ENTRY_CTRL_ST 0xffff0000 /* Steering Tag */ /* CompactPCI Hotswap Register */ @@ -659,6 +660,7 @@ #define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* 64b AtomicOp completion */ #define PCI_EXP_DEVCAP2_ATOMIC_COMP128 0x00000200 /* 128b AtomicOp completion */ #define PCI_EXP_DEVCAP2_LTR 0x00000800 /* Latency tolerance reporting */ +#define PCI_EXP_DEVCAP2_TPH_COMP_MASK 0x00003000 /* TPH completer support */ #define PCI_EXP_DEVCAP2_OBFF_MASK 0x000c0000 /* OBFF support mechanism */ #define PCI_EXP_DEVCAP2_OBFF_MSG 0x00040000 /* New message signaling */ #define PCI_EXP_DEVCAP2_OBFF_WAKE 0x00080000 /* Re-use WAKE# for OBFF */ @@ -678,6 +680,7 @@ #define PCI_EXP_DEVSTA2 0x2a /* Device Status 2 */ #define PCI_CAP_EXP_RC_ENDPOINT_SIZEOF_V2 0x2c /* end of v2 EPs w/o link */ #define PCI_EXP_LNKCAP2 0x2c /* Link Capabilities 2 */ +#define PCI_EXP_LNKCAP2_SLS 0x000000fe /* Supported Link Speeds Vector */ #define PCI_EXP_LNKCAP2_SLS_2_5GB 0x00000002 /* Supported Speed 2.5GT/s */ #define PCI_EXP_LNKCAP2_SLS_5_0GB 0x00000004 /* Supported Speed 5GT/s */ #define PCI_EXP_LNKCAP2_SLS_8_0GB 0x00000008 /* Supported Speed 8GT/s */ @@ -1023,15 +1026,34 @@ #define PCI_DPA_CAP_SUBSTATE_MASK 0x1F /* # substates - 1 */ #define PCI_DPA_BASE_SIZEOF 16 /* size with 0 substates */ +/* TPH Completer Support */ +#define PCI_EXP_DEVCAP2_TPH_COMP_NONE 0x0 /* None */ +#define PCI_EXP_DEVCAP2_TPH_COMP_TPH_ONLY 0x1 /* TPH only */ +#define PCI_EXP_DEVCAP2_TPH_COMP_EXT_TPH 0x3 /* TPH and Extended TPH */ + /* TPH Requester */ #define PCI_TPH_CAP 4 /* capability register */ -#define PCI_TPH_CAP_LOC_MASK 0x600 /* location mask */ -#define PCI_TPH_LOC_NONE 0x000 /* no location */ -#define PCI_TPH_LOC_CAP 0x200 /* in capability */ -#define PCI_TPH_LOC_MSIX 0x400 /* in MSI-X */ -#define PCI_TPH_CAP_ST_MASK 0x07FF0000 /* ST table mask */ -#define PCI_TPH_CAP_ST_SHIFT 16 /* ST table shift */ -#define PCI_TPH_BASE_SIZEOF 0xc /* size with no ST table */ +#define PCI_TPH_CAP_ST_NS 0x00000001 /* No ST Mode Supported */ +#define PCI_TPH_CAP_ST_IV 0x00000002 /* Interrupt Vector Mode Supported */ +#define PCI_TPH_CAP_ST_DS 0x00000004 /* Device Specific Mode Supported */ +#define PCI_TPH_CAP_EXT_TPH 0x00000100 /* Ext TPH Requester Supported */ +#define PCI_TPH_CAP_LOC_MASK 0x00000600 /* ST Table Location */ +#define PCI_TPH_LOC_NONE 0x00000000 /* Not present */ +#define PCI_TPH_LOC_CAP 0x00000200 /* In capability */ +#define PCI_TPH_LOC_MSIX 0x00000400 /* In MSI-X */ +#define PCI_TPH_CAP_ST_MASK 0x07FF0000 /* ST Table Size */ +#define PCI_TPH_CAP_ST_SHIFT 16 /* ST Table Size shift */ +#define PCI_TPH_BASE_SIZEOF 0xc /* Size with no ST table */ + +#define PCI_TPH_CTRL 8 /* control register */ +#define PCI_TPH_CTRL_MODE_SEL_MASK 0x00000007 /* ST Mode Select */ +#define PCI_TPH_ST_NS_MODE 0x0 /* No ST Mode */ +#define PCI_TPH_ST_IV_MODE 0x1 /* Interrupt Vector Mode */ +#define PCI_TPH_ST_DS_MODE 0x2 /* Device Specific Mode */ +#define PCI_TPH_CTRL_REQ_EN_MASK 0x00000300 /* TPH Requester Enable */ +#define PCI_TPH_REQ_DISABLE 0x0 /* No TPH requests allowed */ +#define PCI_TPH_REQ_TPH_ONLY 0x1 /* TPH only requests allowed */ +#define PCI_TPH_REQ_EXT_TPH 0x3 /* Extended TPH requests allowed */ /* Downstream Port Containment */ #define PCI_EXP_DPC_CAP 0x04 /* DPC Capability */ diff --git a/mm/folio-compat.c b/mm/folio-compat.c index 80746182e9e8..1d1832e2a599 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -52,6 +52,12 @@ bool set_page_dirty(struct page *page) } EXPORT_SYMBOL(set_page_dirty); +int set_page_dirty_lock(struct page *page) +{ + return folio_mark_dirty_lock(page_folio(page)); +} +EXPORT_SYMBOL(set_page_dirty_lock); + bool clear_page_dirty_for_io(struct page *page) { return folio_clear_dirty_for_io(page_folio(page)); diff --git a/mm/kasan/kasan_test_rust.rs b/mm/kasan/kasan_test_rust.rs index caa7175964ef..5b34edf30e72 100644 --- a/mm/kasan/kasan_test_rust.rs +++ b/mm/kasan/kasan_test_rust.rs @@ -11,11 +11,12 @@ use kernel::prelude::*; /// drop the vector, and touch it. #[no_mangle] pub extern "C" fn kasan_test_rust_uaf() -> u8 { - let mut v: Vec<u8> = Vec::new(); + let mut v: KVec<u8> = KVec::new(); for _ in 0..4096 { v.push(0x42, GFP_KERNEL).unwrap(); } let ptr: *mut u8 = addr_of_mut!(v[2048]); drop(v); + // SAFETY: Incorrect, on purpose. unsafe { *ptr } } diff --git a/mm/page-writeback.c b/mm/page-writeback.c index fdb89ce85fff..d213ead95675 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2925,25 +2925,25 @@ bool folio_mark_dirty(struct folio *folio) EXPORT_SYMBOL(folio_mark_dirty); /* - * set_page_dirty() is racy if the caller has no reference against - * page->mapping->host, and if the page is unlocked. This is because another - * CPU could truncate the page off the mapping and then free the mapping. + * folio_mark_dirty() is racy if the caller has no reference against + * folio->mapping->host, and if the folio is unlocked. This is because another + * CPU could truncate the folio off the mapping and then free the mapping. * - * Usually, the page _is_ locked, or the caller is a user-space process which + * Usually, the folio _is_ locked, or the caller is a user-space process which * holds a reference on the inode by having an open file. * - * In other cases, the page should be locked before running set_page_dirty(). + * In other cases, the folio should be locked before running folio_mark_dirty(). */ -int set_page_dirty_lock(struct page *page) +bool folio_mark_dirty_lock(struct folio *folio) { - int ret; + bool ret; - lock_page(page); - ret = set_page_dirty(page); - unlock_page(page); + folio_lock(folio); + ret = folio_mark_dirty(folio); + folio_unlock(folio); return ret; } -EXPORT_SYMBOL(set_page_dirty_lock); +EXPORT_SYMBOL(folio_mark_dirty_lock); /* * This cancels just the dirty bit on the kernel page itself, it does NOT diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 1bd3e531b0e0..059f6ef1ad18 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -1427,7 +1427,9 @@ static int c_show(struct seq_file *m, void *p) seq_printf(m, "# expiry=%lld refcnt=%d flags=%lx\n", convert_to_wallclock(cp->expiry_time), kref_read(&cp->ref), cp->flags); - cache_get(cp); + if (!cache_get_rcu(cp)) + return 0; + if (cache_check(cd, cp, NULL)) /* cache_check does a cache_put on failure */ seq_puts(m, "# "); diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c index 58ae6ec4f25b..415c0310101f 100644 --- a/net/sunrpc/xprtrdma/svc_rdma.c +++ b/net/sunrpc/xprtrdma/svc_rdma.c @@ -233,25 +233,34 @@ static int svc_rdma_proc_init(void) rc = percpu_counter_init(&svcrdma_stat_read, 0, GFP_KERNEL); if (rc) - goto out_err; + goto err; rc = percpu_counter_init(&svcrdma_stat_recv, 0, GFP_KERNEL); if (rc) - goto out_err; + goto err_read; rc = percpu_counter_init(&svcrdma_stat_sq_starve, 0, GFP_KERNEL); if (rc) - goto out_err; + goto err_recv; rc = percpu_counter_init(&svcrdma_stat_write, 0, GFP_KERNEL); if (rc) - goto out_err; + goto err_sq; svcrdma_table_header = register_sysctl("sunrpc/svc_rdma", svcrdma_parm_table); + if (!svcrdma_table_header) + goto err_write; + return 0; -out_err: +err_write: + rc = -ENOMEM; + percpu_counter_destroy(&svcrdma_stat_write); +err_sq: percpu_counter_destroy(&svcrdma_stat_sq_starve); +err_recv: percpu_counter_destroy(&svcrdma_stat_recv); +err_read: percpu_counter_destroy(&svcrdma_stat_read); +err: return rc; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index ae3fb9bc8a21..292022f0976e 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -493,7 +493,13 @@ static bool xdr_check_write_chunk(struct svc_rdma_recv_ctxt *rctxt) if (xdr_stream_decode_u32(&rctxt->rc_stream, &segcount)) return false; - /* A bogus segcount causes this buffer overflow check to fail. */ + /* Before trusting the segcount value enough to use it in + * a computation, perform a simple range check. This is an + * arbitrary but sensible limit (ie, not architectural). + */ + if (unlikely(segcount > RPCSVC_MAXPAGES)) + return false; + p = xdr_inline_decode(&rctxt->rc_stream, segcount * rpcrdma_segment_maxsz * sizeof(*p)); return p != NULL; diff --git a/rust/Makefile b/rust/Makefile index 99db963637e6..01cf8e6ba3b3 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -3,7 +3,7 @@ # Where to place rustdoc generated documentation rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc -obj-$(CONFIG_RUST) += core.o compiler_builtins.o +obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o always-$(CONFIG_RUST) += exports_core_generated.h # Missing prototypes are expected in the helpers since these are exported @@ -15,8 +15,8 @@ always-$(CONFIG_RUST) += libmacros.so no-clean-files += libmacros.so always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs -obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o -always-$(CONFIG_RUST) += exports_alloc_generated.h exports_helpers_generated.h \ +obj-$(CONFIG_RUST) += bindings.o kernel.o +always-$(CONFIG_RUST) += exports_helpers_generated.h \ exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs @@ -55,15 +55,10 @@ endif core-cfgs = \ --cfg no_fp_fmt_parse -alloc-cfgs = \ - --cfg no_global_oom_handling \ - --cfg no_rc \ - --cfg no_sync - quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \ + $(RUSTDOC) $(filter-out $(skip_flags),$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ $(rustc_target_flags) -L$(objtree)/$(obj) \ -Zunstable-options --generate-link-to-definition \ --output $(rustdoc_output) \ @@ -83,7 +78,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # command-like flags to solve the issue. Meanwhile, we use the non-custom case # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ - rustdoc-alloc rustdoc-kernel + rustdoc-kernel $(Q)cp $(srctree)/Documentation/images/logo.svg $(rustdoc_output)/static.files/ $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(rustdoc_output)/static.files/ $(Q)find $(rustdoc_output) -name '*.html' -type f -print0 | xargs -0 sed -Ei \ @@ -100,6 +95,9 @@ rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ rustdoc-macros: $(src)/macros/lib.rs FORCE +$(call if_changed,rustdoc) +# Starting with Rust 1.82.0, skipping `-Wrustdoc::unescaped_backticks` should +# not be needed -- see https://github.com/rust-lang/rust/pull/128307. +rustdoc-core: private skip_flags = -Wrustdoc::unescaped_backticks rustdoc-core: private rustc_target_flags = $(core-cfgs) rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE +$(call if_changed,rustdoc) @@ -107,20 +105,14 @@ rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE +$(call if_changed,rustdoc) -# We need to allow `rustdoc::broken_intra_doc_links` because some -# `no_global_oom_handling` functions refer to non-`no_global_oom_handling` -# functions. Ideally `rustdoc` would have a way to distinguish broken links -# due to things that are "configured out" vs. entirely non-existing ones. -rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \ - -Arustdoc::broken_intra_doc_links -rustdoc-alloc: $(RUST_LIB_SRC)/alloc/src/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE +rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE +$(call if_changed,rustdoc) -rustdoc-kernel: private rustc_target_flags = --extern alloc \ +rustdoc-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ --extern bindings --extern uapi -rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ - rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ +rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \ + rustdoc-compiler_builtins $(obj)/libmacros.so \ $(obj)/bindings.o FORCE +$(call if_changed,rustdoc) @@ -137,19 +129,33 @@ quiet_cmd_rustc_test_library = RUSTC TL $< rusttestlib-build_error: $(src)/build_error.rs FORCE +$(call if_changed,rustc_test_library) +rusttestlib-ffi: $(src)/ffi.rs FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-macros: private rustc_target_flags = --extern proc_macro rusttestlib-macros: private rustc_test_library_proc = yes rusttestlib-macros: $(src)/macros/lib.rs FORCE +$(call if_changed,rustc_test_library) -rusttestlib-bindings: $(src)/bindings/lib.rs FORCE +rusttestlib-kernel: private rustc_target_flags = --extern ffi \ + --extern build_error --extern macros \ + --extern bindings --extern uapi +rusttestlib-kernel: $(src)/kernel/lib.rs \ + rusttestlib-bindings rusttestlib-uapi rusttestlib-build_error \ + $(obj)/libmacros.so $(obj)/bindings.o FORCE + +$(call if_changed,rustc_test_library) + +rusttestlib-bindings: private rustc_target_flags = --extern ffi +rusttestlib-bindings: $(src)/bindings/lib.rs rusttestlib-ffi FORCE +$(call if_changed,rustc_test_library) -rusttestlib-uapi: $(src)/uapi/lib.rs FORCE +rusttestlib-uapi: private rustc_target_flags = --extern ffi +rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi FORCE +$(call if_changed,rustc_test_library) quiet_cmd_rustdoc_test = RUSTDOC T $< cmd_rustdoc_test = \ + RUST_MODFILE=test.rs \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_common_flags) \ @$(objtree)/include/generated/rustc_cfg \ @@ -164,7 +170,7 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_flags) \ - -L$(objtree)/$(obj) --extern alloc --extern kernel \ + -L$(objtree)/$(obj) --extern ffi --extern kernel \ --extern build_error --extern macros \ --extern bindings --extern uapi \ --no-run --crate-name kernel -Zunstable-options \ @@ -194,19 +200,20 @@ quiet_cmd_rustc_test = RUSTC T $< rusttest: rusttest-macros rusttest-kernel -rusttest-macros: private rustc_target_flags = --extern proc_macro +rusttest-macros: private rustc_target_flags = --extern proc_macro \ + --extern macros --extern kernel rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro -rusttest-macros: $(src)/macros/lib.rs FORCE +rusttest-macros: $(src)/macros/lib.rs \ + rusttestlib-macros rusttestlib-kernel FORCE +$(call if_changed,rustc_test) +$(call if_changed,rustdoc_test) -rusttest-kernel: private rustc_target_flags = --extern alloc \ +rusttest-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern bindings --extern uapi -rusttest-kernel: $(src)/kernel/lib.rs \ +rusttest-kernel: $(src)/kernel/lib.rs rusttestlib-ffi rusttestlib-kernel \ rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ rusttestlib-uapi FORCE +$(call if_changed,rustc_test) - +$(call if_changed,rustc_test_library) ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) @@ -267,12 +274,16 @@ else bindgen_c_flags_lto = $(bindgen_c_flags) endif -bindgen_c_flags_final = $(bindgen_c_flags_lto) -D__BINDGEN__ +# `-fno-builtin` is passed to avoid `bindgen` from using `clang` builtin +# prototypes for functions like `memcpy` -- if this flag is not passed, +# `bindgen`-generated prototypes use `c_ulong` or `c_uint` depending on +# architecture instead of generating `usize`. +bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) \ - --use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \ + --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) @@ -313,9 +324,6 @@ quiet_cmd_exports = EXPORTS $@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(call if_changed,exports) -$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE - $(call if_changed,exports) - # Even though Rust kernel modules should never use the bindings directly, # symbols from the `bindings` crate and the C helpers need to be exported # because Rust generics and inlined functions may not get their code generated @@ -362,7 +370,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L rust-analyzer: $(Q)$(srctree)/scripts/generate_rust_analyzer.py \ - --cfgs='core=$(core-cfgs)' --cfgs='alloc=$(alloc-cfgs)' \ + --cfgs='core=$(core-cfgs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(KBUILD_EXTMOD) > \ $(if $(KBUILD_EXTMOD),$(extmod_prefix),$(objtree))/rust-project.json @@ -400,29 +408,28 @@ $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE +$(call if_changed_rule,rustc_library) -$(obj)/alloc.o: private skip_clippy = 1 -$(obj)/alloc.o: private skip_flags = -Wunreachable_pub -$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs) -$(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE +$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) -$(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE +$(obj)/ffi.o: $(src)/ffi.rs $(obj)/compiler_builtins.o FORCE +$(call if_changed_rule,rustc_library) +$(obj)/bindings.o: private rustc_target_flags = --extern ffi $(obj)/bindings.o: $(src)/bindings/lib.rs \ - $(obj)/compiler_builtins.o \ + $(obj)/ffi.o \ $(obj)/bindings/bindings_generated.rs \ $(obj)/bindings/bindings_helpers_generated.rs FORCE +$(call if_changed_rule,rustc_library) +$(obj)/uapi.o: private rustc_target_flags = --extern ffi $(obj)/uapi.o: $(src)/uapi/lib.rs \ - $(obj)/compiler_builtins.o \ + $(obj)/ffi.o \ $(obj)/uapi/uapi_generated.rs FORCE +$(call if_changed_rule,rustc_library) -$(obj)/kernel.o: private rustc_target_flags = --extern alloc \ +$(obj)/kernel.o: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern bindings --extern uapi -$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o \ $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE +$(call if_changed_rule,rustc_library) diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index b7c7483123b7..0f96af8b9a7f 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 +# We want to map these types to `isize`/`usize` manually, instead of +# define them as `int`/`long` depending on platform bitwidth. +--blocklist-type __kernel_s?size_t +--blocklist-type __kernel_ptrdiff_t + --opaque-type xregs_state --opaque-type desc_struct --opaque-type arch_lbr_state diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 0e9fee0a4a27..54bc8a8c5c88 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -40,4 +40,5 @@ const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; const gfp_t RUST_CONST_HELPER_GFP_NOWAIT = GFP_NOWAIT; const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; +const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 93a1a3fc97bc..014af0d1fc70 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -25,7 +25,13 @@ )] #[allow(dead_code)] +#[allow(clippy::undocumented_unsafe_blocks)] mod bindings_raw { + // Manual definition for blocklisted types. + type __kernel_size_t = usize; + type __kernel_ssize_t = isize; + type __kernel_ptrdiff_t = isize; + // Use glob import here to expose all helpers. // Symbols defined within the module will take precedence to the glob import. pub use super::bindings_helper::*; diff --git a/rust/exports.c b/rust/exports.c index e5695f3b45b7..587f0e776aba 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -3,9 +3,9 @@ * A hack to export Rust symbols for loadable modules without having to redo * the entire `include/linux/export.h` logic in Rust. * - * This requires the Rust's new/future `v0` mangling scheme because the default - * one ("legacy") uses invalid characters for C identifiers (thus we cannot use - * the `EXPORT_SYMBOL_*` macros). + * This requires Rust's new/future `v0` mangling scheme because the default one + * ("legacy") uses invalid characters for C identifiers (thus we cannot use the + * `EXPORT_SYMBOL_*` macros). * * All symbols are exported as GPL-only to guarantee no GPL-only feature is * accidentally exposed. @@ -16,7 +16,6 @@ #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) #include "exports_core_generated.h" -#include "exports_alloc_generated.h" #include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" diff --git a/rust/ffi.rs b/rust/ffi.rs new file mode 100644 index 000000000000..be153c4d551b --- /dev/null +++ b/rust/ffi.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Foreign function interface (FFI) types. +//! +//! This crate provides mapping from C primitive types to Rust ones. +//! +//! The Rust [`core`] crate provides [`core::ffi`], which maps integer types to the platform default +//! C ABI. The kernel does not use [`core::ffi`], so it can customise the mapping that deviates from +//! the platform default. + +#![no_std] + +pub use core::ffi::*; diff --git a/rust/helpers/build_bug.c b/rust/helpers/build_bug.c index e994f7b5928c..44e579488037 100644 --- a/rust/helpers/build_bug.c +++ b/rust/helpers/build_bug.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/errname.h> const char *rust_helper_errname(int err) diff --git a/rust/helpers/err.c b/rust/helpers/err.c index be3d45ef78a2..544c7cb86632 100644 --- a/rust/helpers/err.c +++ b/rust/helpers/err.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/err.h> -#include <linux/export.h> __force void *rust_helper_ERR_PTR(long err) { diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 463b970154de..dcf827a61b52 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -18,6 +18,7 @@ #include "kunit.c" #include "mutex.c" #include "page.c" +#include "pid_namespace.c" #include "rbtree.c" #include "refcount.c" #include "security.c" @@ -26,5 +27,6 @@ #include "spinlock.c" #include "task.c" #include "uaccess.c" +#include "vmalloc.c" #include "wait.c" #include "workqueue.c" diff --git a/rust/helpers/kunit.c b/rust/helpers/kunit.c index 9d725067eb3b..b85a4d394c11 100644 --- a/rust/helpers/kunit.c +++ b/rust/helpers/kunit.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <kunit/test-bug.h> -#include <linux/export.h> struct kunit *rust_helper_kunit_get_current_test(void) { diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c index a17ca8cdb50c..7e00680958ef 100644 --- a/rust/helpers/mutex.c +++ b/rust/helpers/mutex.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/mutex.h> void rust_helper_mutex_lock(struct mutex *lock) diff --git a/rust/helpers/pid_namespace.c b/rust/helpers/pid_namespace.c new file mode 100644 index 000000000000..f41482bdec9a --- /dev/null +++ b/rust/helpers/pid_namespace.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/pid_namespace.h> +#include <linux/cleanup.h> + +struct pid_namespace *rust_helper_get_pid_ns(struct pid_namespace *ns) +{ + return get_pid_ns(ns); +} + +void rust_helper_put_pid_ns(struct pid_namespace *ns) +{ + put_pid_ns(ns); +} + +/* Get a reference on a task's pid namespace. */ +struct pid_namespace *rust_helper_task_get_pid_ns(struct task_struct *task) +{ + struct pid_namespace *pid_ns; + + guard(rcu)(); + pid_ns = task_active_pid_ns(task); + if (pid_ns) + get_pid_ns(pid_ns); + return pid_ns; +} diff --git a/rust/helpers/refcount.c b/rust/helpers/refcount.c index f47afc148ec3..d6adbd2e45a1 100644 --- a/rust/helpers/refcount.c +++ b/rust/helpers/refcount.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/refcount.h> refcount_t rust_helper_REFCOUNT_INIT(int n) diff --git a/rust/helpers/signal.c b/rust/helpers/signal.c index 63c407f80c26..1a6bbe9438e2 100644 --- a/rust/helpers/signal.c +++ b/rust/helpers/signal.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/sched/signal.h> int rust_helper_signal_pending(struct task_struct *t) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index f043e087f9d6..a842bfbddcba 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -7,3 +7,9 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) { return krealloc(objp, new_size, flags); } + +void * __must_check __realloc_size(2) +rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +{ + return kvrealloc(p, size, flags); +} diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 92f7fc418425..5971fdf6f755 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/spinlock.h> void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, @@ -26,3 +25,8 @@ void rust_helper_spin_unlock(spinlock_t *lock) { spin_unlock(lock); } + +int rust_helper_spin_trylock(spinlock_t *lock) +{ + return spin_trylock(lock); +} diff --git a/rust/helpers/task.c b/rust/helpers/task.c index 7d66487db831..31c33ea2dce6 100644 --- a/rust/helpers/task.c +++ b/rust/helpers/task.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/sched/task.h> struct task_struct *rust_helper_get_current(void) diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c new file mode 100644 index 000000000000..80d34501bbc0 --- /dev/null +++ b/rust/helpers/vmalloc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/vmalloc.h> + +void * __must_check __realloc_size(2) +rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +{ + return vrealloc(p, size, flags); +} diff --git a/rust/helpers/wait.c b/rust/helpers/wait.c index c7336bbf2750..ae48e33d9da3 100644 --- a/rust/helpers/wait.c +++ b/rust/helpers/wait.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/wait.h> void rust_helper_init_wait(struct wait_queue_entry *wq_entry) diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index f59427acc323..b2b82753509b 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/export.h> #include <linux/workqueue.h> void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func, diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 1966bd407017..f2f7f3a53d29 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -1,23 +1,41 @@ // SPDX-License-Identifier: GPL-2.0 -//! Extensions to the [`alloc`] crate. +//! Implementation of the kernel's memory allocation infrastructure. -#[cfg(not(test))] -#[cfg(not(testlib))] -mod allocator; -pub mod box_ext; -pub mod vec_ext; +#[cfg(not(any(test, testlib)))] +pub mod allocator; +pub mod kbox; +pub mod kvec; +pub mod layout; + +#[cfg(any(test, testlib))] +pub mod allocator_test; + +#[cfg(any(test, testlib))] +pub use self::allocator_test as allocator; + +pub use self::kbox::Box; +pub use self::kbox::KBox; +pub use self::kbox::KVBox; +pub use self::kbox::VBox; + +pub use self::kvec::IntoIter; +pub use self::kvec::KVVec; +pub use self::kvec::KVec; +pub use self::kvec::VVec; +pub use self::kvec::Vec; /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; +use core::{alloc::Layout, ptr::NonNull}; /// Flags to be used when allocating memory. /// /// They can be combined with the operators `|`, `&`, and `!`. /// /// Values can be used from the [`flags`] module. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq)] pub struct Flags(u32); impl Flags { @@ -25,6 +43,11 @@ impl Flags { pub(crate) fn as_raw(self) -> u32 { self.0 } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } } impl core::ops::BitOr for Flags { @@ -85,4 +108,117 @@ pub mod flags { /// use any filesystem callback. It is very likely to fail to allocate memory, even for very /// small allocations. pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); + + /// Suppresses allocation failure reports. + /// + /// This is normally or'd with other flags. + pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); +} + +/// The kernel's [`Allocator`] trait. +/// +/// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described +/// via [`Layout`]. +/// +/// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on +/// an object instance. +/// +/// In order to be able to support `#[derive(SmartPointer)]` later on, we need to avoid a design +/// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind +/// of `self` parameter. +/// +/// # Safety +/// +/// - A memory allocation returned from an allocator must remain valid until it is explicitly freed. +/// +/// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`] +/// function of the same type. +/// +/// - Implementers must ensure that all trait functions abide by the guarantees documented in the +/// `# Guarantees` sections. +pub unsafe trait Allocator { + /// Allocate memory based on `layout` and `flags`. + /// + /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout + /// constraints (i.e. minimum size and alignment as specified by `layout`). + /// + /// This function is equivalent to `realloc` when called with `None`. + /// + /// # Guarantees + /// + /// When the return value is `Ok(ptr)`, then `ptr` is + /// - valid for reads and writes for `layout.size()` bytes, until it is passed to + /// [`Allocator::free`] or [`Allocator::realloc`], + /// - aligned to `layout.align()`, + /// + /// Additionally, `Flags` are honored as documented in + /// <https://docs.kernel.org/core-api/mm-api.html#mm-api-gfp-flags>. + fn alloc(layout: Layout, flags: Flags) -> Result<NonNull<[u8]>, AllocError> { + // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a + // new memory allocation. + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) } + } + + /// Re-allocate an existing memory allocation to satisfy the requested `layout`. + /// + /// If the requested size is zero, `realloc` behaves equivalent to `free`. + /// + /// If the requested size is larger than the size of the existing allocation, a successful call + /// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but + /// may also be larger. + /// + /// If the requested size is smaller than the size of the existing allocation, `realloc` may or + /// may not shrink the buffer; this is implementation specific to the allocator. + /// + /// On allocation failure, the existing buffer, if any, remains valid. + /// + /// The buffer is represented as `NonNull<[u8]>`. + /// + /// # Safety + /// + /// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation + /// created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a + /// pointer returned by this [`Allocator`]. + /// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and + /// `old_layout` is ignored. + /// - `old_layout` must match the `Layout` the allocation has been created with. + /// + /// # Guarantees + /// + /// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then + /// it additionally guarantees that: + /// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new + /// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] == + /// p[0..min(layout.size(), old_layout.size())]`. + /// - when the return value is `Err(AllocError)`, then `ptr` is still valid. + unsafe fn realloc( + ptr: Option<NonNull<u8>>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result<NonNull<[u8]>, AllocError>; + + /// Free an existing memory allocation. + /// + /// # Safety + /// + /// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`]; + /// if `old_layout` is zero-sized `p` does not need to be a pointer returned by this + /// [`Allocator`]. + /// - `layout` must match the `Layout` the allocation has been created with. + /// - The memory allocation at `ptr` must never again be read from or written to. + unsafe fn free(ptr: NonNull<u8>, layout: Layout) { + // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this + // allocator. We are passing a `Layout` with the smallest possible alignment, so it is + // smaller than or equal to the alignment previously used with this allocation. + let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; + } +} + +/// Returns a properly aligned dangling pointer from the given `layout`. +pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull<u8> { + let ptr = layout.align() as *mut u8; + + // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. + unsafe { NonNull::new_unchecked(ptr) } } diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index e6ea601f38c6..439985e29fbc 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,74 +1,188 @@ // SPDX-License-Identifier: GPL-2.0 //! Allocator support. +//! +//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" +//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the +//! typical application of the different kernel allocators. +//! +//! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> -use super::{flags::*, Flags}; -use core::alloc::{GlobalAlloc, Layout}; +use super::Flags; +use core::alloc::Layout; use core::ptr; +use core::ptr::NonNull; -struct KernelAllocator; +use crate::alloc::{AllocError, Allocator}; +use crate::bindings; +use crate::pr_warn; -/// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. +/// The contiguous kernel allocator. /// -/// # Safety +/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also +/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. /// -/// - `ptr` can be either null or a pointer which has been allocated by this allocator. -/// - `new_layout` must have a non-zero size. -pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { +/// For more details see [self]. +pub struct Kmalloc; + +/// The virtually contiguous kernel allocator. +/// +/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel +/// virtual space. It is typically used for large allocations. The memory allocated with this +/// allocator is not physically contiguous. +/// +/// For more details see [self]. +pub struct Vmalloc; + +/// The kvmalloc kernel allocator. +/// +/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon +/// failure. This allocator is typically used when the size for the requested allocation is not +/// known and may exceed the capabilities of `Kmalloc`. +/// +/// For more details see [self]. +pub struct KVmalloc; + +/// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. +fn aligned_size(new_layout: Layout) -> usize { // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. let layout = new_layout.pad_to_align(); // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()` // which together with the slab guarantees means the `krealloc` will return a properly aligned // object (see comments in `kmalloc()` for more information). - let size = layout.size(); - - // SAFETY: - // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the - // function safety requirement. - // - `size` is greater than 0 since it's from `layout.size()` (which cannot be zero according - // to the function safety requirement) - unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } + layout.size() } -unsafe impl GlobalAlloc for KernelAllocator { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety - // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } - } +/// # Invariants +/// +/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. +struct ReallocFunc( + unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void, +); - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - unsafe { - bindings::kfree(ptr as *const core::ffi::c_void); - } - } +impl ReallocFunc { + // INVARIANT: `krealloc` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc); - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - // SAFETY: - // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not - // overflow `isize` by the function safety requirement. - // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). - let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + // INVARIANT: `vrealloc` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc); + + // INVARIANT: `kvrealloc` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc); + + /// # Safety + /// + /// This method has the same safety requirements as [`Allocator::realloc`]. + /// + /// # Guarantees + /// + /// This method has the same guarantees as `Allocator::realloc`. Additionally + /// - it accepts any pointer to a valid memory allocation allocated by this function. + /// - memory allocated by this function remains valid until it is passed to this function. + unsafe fn call( + &self, + ptr: Option<NonNull<u8>>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result<NonNull<[u8]>, AllocError> { + let size = aligned_size(layout); + let ptr = match ptr { + Some(ptr) => { + if old_layout.size() == 0 { + ptr::null() + } else { + ptr.as_ptr() + } + } + None => ptr::null(), + }; // SAFETY: - // - `ptr` is either null or a pointer allocated by this allocator by the function safety - // requirement. - // - the size of `layout` is not zero because `new_size` is not zero by the function safety - // requirement. - unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that + // `ptr` is NULL or valid. + // - `ptr` is either NULL or valid by the safety requirements of this function. + // + // GUARANTEE: + // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. + // - Those functions provide the guarantees of this function. + let raw_ptr = unsafe { + // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. + self.0(ptr.cast(), size, flags.0).cast() + }; + + let ptr = if size == 0 { + crate::alloc::dangling_from_layout(layout) + } else { + NonNull::new(raw_ptr).ok_or(AllocError)? + }; + + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Kmalloc { + #[inline] + unsafe fn realloc( + ptr: Option<NonNull<u8>>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result<NonNull<[u8]>, AllocError> { + // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } } +} + +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for Vmalloc { + #[inline] + unsafe fn realloc( + ptr: Option<NonNull<u8>>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result<NonNull<[u8]>, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety - // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } } } -#[global_allocator] -static ALLOCATOR: KernelAllocator = KernelAllocator; +// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation is OK, +// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. +unsafe impl Allocator for KVmalloc { + #[inline] + unsafe fn realloc( + ptr: Option<NonNull<u8>>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result<NonNull<[u8]>, AllocError> { + // TODO: Support alignments larger than PAGE_SIZE. + if layout.align() > bindings::PAGE_SIZE { + pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); + return Err(AllocError); + } -// See <https://github.com/rust-lang/rust/pull/86844>. -#[no_mangle] -static __rust_no_alloc_shim_is_unstable: u8 = 0; + // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously + // allocated with this `Allocator`. + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + } +} diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs new file mode 100644 index 000000000000..e3240d16040b --- /dev/null +++ b/rust/kernel/alloc/allocator_test.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users +//! of those types (e.g. `CString`) use kernel allocators for instantiation. +//! +//! In order to allow userspace test cases to make use of such types as well, implement the +//! `Cmalloc` allocator within the allocator_test module and type alias all kernel allocators to +//! `Cmalloc`. The `Cmalloc` allocator uses libc's `realloc()` function as allocator backend. + +#![allow(missing_docs)] + +use super::{flags::*, AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::cmp; +use core::ptr; +use core::ptr::NonNull; + +/// The userspace allocator based on libc. +pub struct Cmalloc; + +pub type Kmalloc = Cmalloc; +pub type Vmalloc = Kmalloc; +pub type KVmalloc = Kmalloc; + +extern "C" { + #[link_name = "aligned_alloc"] + fn libc_aligned_alloc(align: usize, size: usize) -> *mut crate::ffi::c_void; + + #[link_name = "free"] + fn libc_free(ptr: *mut crate::ffi::c_void); +} + +// SAFETY: +// - memory remains valid until it is explicitly freed, +// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK, +// - `realloc` provides the guarantees as provided in the `# Guarantees` section. +unsafe impl Allocator for Cmalloc { + unsafe fn realloc( + ptr: Option<NonNull<u8>>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result<NonNull<[u8]>, AllocError> { + let src = match ptr { + Some(src) => { + if old_layout.size() == 0 { + ptr::null_mut() + } else { + src.as_ptr() + } + } + None => ptr::null_mut(), + }; + + if layout.size() == 0 { + // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` + unsafe { libc_free(src.cast()) }; + + return Ok(NonNull::slice_from_raw_parts( + crate::alloc::dangling_from_layout(layout), + 0, + )); + } + + // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or + // exceeds the given size and alignment requirements. + let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) } as *mut u8; + let dst = NonNull::new(dst).ok_or(AllocError)?; + + if flags.contains(__GFP_ZERO) { + // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new` + // guarantee that `dst` points to memory of at least `layout.size()` bytes. + unsafe { dst.as_ptr().write_bytes(0, layout.size()) }; + } + + if !src.is_null() { + // SAFETY: + // - `src` has previously been allocated with this `Allocator`; `dst` has just been + // newly allocated, hence the memory regions do not overlap. + // - both` src` and `dst` are properly aligned and valid for reads and writes + unsafe { + ptr::copy_nonoverlapping( + src, + dst.as_ptr(), + cmp::min(layout.size(), old_layout.size()), + ) + }; + } + + // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` + unsafe { libc_free(src.cast()) }; + + Ok(NonNull::slice_from_raw_parts(dst, layout.size())) + } +} diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs deleted file mode 100644 index 7009ad78d4e0..000000000000 --- a/rust/kernel/alloc/box_ext.rs +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Extensions to [`Box`] for fallible allocations. - -use super::{AllocError, Flags}; -use alloc::boxed::Box; -use core::{mem::MaybeUninit, ptr, result::Result}; - -/// Extensions to [`Box`]. -pub trait BoxExt<T>: Sized { - /// Allocates a new box. - /// - /// The allocation may fail, in which case an error is returned. - fn new(x: T, flags: Flags) -> Result<Self, AllocError>; - - /// Allocates a new uninitialised box. - /// - /// The allocation may fail, in which case an error is returned. - fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError>; - - /// Drops the contents, but keeps the allocation. - /// - /// # Examples - /// - /// ``` - /// use kernel::alloc::{flags, box_ext::BoxExt}; - /// let value = Box::new([0; 32], flags::GFP_KERNEL)?; - /// assert_eq!(*value, [0; 32]); - /// let mut value = Box::drop_contents(value); - /// // Now we can re-use `value`: - /// value.write([1; 32]); - /// // SAFETY: We just wrote to it. - /// let value = unsafe { value.assume_init() }; - /// assert_eq!(*value, [1; 32]); - /// # Ok::<(), Error>(()) - /// ``` - fn drop_contents(this: Self) -> Box<MaybeUninit<T>>; -} - -impl<T> BoxExt<T> for Box<T> { - fn new(x: T, flags: Flags) -> Result<Self, AllocError> { - let mut b = <Self as BoxExt<_>>::new_uninit(flags)?; - b.write(x); - // SAFETY: We just wrote to it. - Ok(unsafe { b.assume_init() }) - } - - #[cfg(any(test, testlib))] - fn new_uninit(_flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> { - Ok(Box::new_uninit()) - } - - #[cfg(not(any(test, testlib)))] - fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> { - let ptr = if core::mem::size_of::<MaybeUninit<T>>() == 0 { - core::ptr::NonNull::<_>::dangling().as_ptr() - } else { - let layout = core::alloc::Layout::new::<MaybeUninit<T>>(); - - // SAFETY: Memory is being allocated (first arg is null). The only other source of - // safety issues is sleeping on atomic context, which is addressed by klint. Lastly, - // the type is not a SZT (checked above). - let ptr = - unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) }; - if ptr.is_null() { - return Err(AllocError); - } - - ptr.cast::<MaybeUninit<T>>() - }; - - // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For - // zero-sized types, we use `NonNull::dangling`. - Ok(unsafe { Box::from_raw(ptr) }) - } - - fn drop_contents(this: Self) -> Box<MaybeUninit<T>> { - let ptr = Box::into_raw(this); - // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. - unsafe { ptr::drop_in_place(ptr) }; - - // CAST: `MaybeUninit<T>` is a transparent wrapper of `T`. - let ptr = ptr.cast::<MaybeUninit<T>>(); - - // SAFETY: `ptr` is valid for writes, because it came from `Box::into_raw` and it is valid for - // reads, since the pointer came from `Box::into_raw` and the type is `MaybeUninit<T>`. - unsafe { Box::from_raw(ptr) } - } -} diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs new file mode 100644 index 000000000000..9ce414361c2c --- /dev/null +++ b/rust/kernel/alloc/kbox.rs @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Box`]. + +#[allow(unused_imports)] // Used in doc comments. +use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; +use super::{AllocError, Allocator, Flags}; +use core::alloc::Layout; +use core::fmt; +use core::marker::PhantomData; +use core::mem::ManuallyDrop; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::pin::Pin; +use core::ptr::NonNull; +use core::result::Result; + +use crate::init::{InPlaceInit, InPlaceWrite, Init, PinInit}; +use crate::types::ForeignOwnable; + +/// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. +/// +/// This is the kernel's version of the Rust stdlib's `Box`. There are several differences, +/// for example no `noalias` attribute is emitted and partially moving out of a `Box` is not +/// supported. There are also several API differences, e.g. `Box` always requires an [`Allocator`] +/// implementation to be passed as generic, page [`Flags`] when allocating memory and all functions +/// that may allocate memory are fallible. +/// +/// `Box` works with any of the kernel's allocators, e.g. [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]. +/// There are aliases for `Box` with these allocators ([`KBox`], [`VBox`], [`KVBox`]). +/// +/// When dropping a [`Box`], the value is also dropped and the heap memory is automatically freed. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::<u64>::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KBox::<Huge>::new_uninit(GFP_KERNEL | __GFP_NOWARN).is_err()); +/// ``` +/// +/// ``` +/// # use kernel::bindings; +/// const SIZE: usize = bindings::KMALLOC_MAX_SIZE as usize + 1; +/// struct Huge([u8; SIZE]); +/// +/// assert!(KVBox::<Huge>::new_uninit(GFP_KERNEL).is_ok()); +/// ``` +/// +/// # Invariants +/// +/// `self.0` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +#[repr(transparent)] +pub struct Box<T: ?Sized, A: Allocator>(NonNull<T>, PhantomData<A>); + +/// Type alias for [`Box`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KBox<T> = Box<T, super::allocator::Kmalloc>; + +/// Type alias for [`Box`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = VBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type VBox<T> = Box<T, super::allocator::Vmalloc>; + +/// Type alias for [`Box`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let b = KVBox::new(24_u64, GFP_KERNEL)?; +/// +/// assert_eq!(*b, 24_u64); +/// # Ok::<(), Error>(()) +/// ``` +pub type KVBox<T> = Box<T, super::allocator::KVmalloc>; + +// SAFETY: `Box` is `Send` if `T` is `Send` because the `Box` owns a `T`. +unsafe impl<T, A> Send for Box<T, A> +where + T: Send + ?Sized, + A: Allocator, +{ +} + +// SAFETY: `Box` is `Sync` if `T` is `Sync` because the `Box` owns a `T`. +unsafe impl<T, A> Sync for Box<T, A> +where + T: Sync + ?Sized, + A: Allocator, +{ +} + +impl<T, A> Box<T, A> +where + T: ?Sized, + A: Allocator, +{ + /// Creates a new `Box<T, A>` from a raw pointer. + /// + /// # Safety + /// + /// For non-ZSTs, `raw` must point at an allocation allocated with `A` that is sufficiently + /// aligned for and holds a valid `T`. The caller passes ownership of the allocation to the + /// `Box`. + /// + /// For ZSTs, `raw` must be a dangling, well aligned pointer. + #[inline] + pub const unsafe fn from_raw(raw: *mut T) -> Self { + // INVARIANT: Validity of `raw` is guaranteed by the safety preconditions of this function. + // SAFETY: By the safety preconditions of this function, `raw` is not a NULL pointer. + Self(unsafe { NonNull::new_unchecked(raw) }, PhantomData) + } + + /// Consumes the `Box<T, A>` and returns a raw pointer. + /// + /// This will not run the destructor of `T` and for non-ZSTs the allocation will stay alive + /// indefinitely. Use [`Box::from_raw`] to recover the [`Box`], drop the value and free the + /// allocation, if any. + /// + /// # Examples + /// + /// ``` + /// let x = KBox::new(24, GFP_KERNEL)?; + /// let ptr = KBox::into_raw(x); + /// // SAFETY: `ptr` comes from a previous call to `KBox::into_raw`. + /// let x = unsafe { KBox::from_raw(ptr) }; + /// + /// assert_eq!(*x, 24); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub fn into_raw(b: Self) -> *mut T { + ManuallyDrop::new(b).0.as_ptr() + } + + /// Consumes and leaks the `Box<T, A>` and returns a mutable reference. + /// + /// See [`Box::into_raw`] for more details. + #[inline] + pub fn leak<'a>(b: Self) -> &'a mut T { + // SAFETY: `Box::into_raw` always returns a properly aligned and dereferenceable pointer + // which points to an initialized instance of `T`. + unsafe { &mut *Box::into_raw(b) } + } +} + +impl<T, A> Box<MaybeUninit<T>, A> +where + A: Allocator, +{ + /// Converts a `Box<MaybeUninit<T>, A>` to a `Box<T, A>`. + /// + /// It is undefined behavior to call this function while the value inside of `b` is not yet + /// fully initialized. + /// + /// # Safety + /// + /// Callers must ensure that the value inside of `b` is in an initialized state. + pub unsafe fn assume_init(self) -> Box<T, A> { + let raw = Self::into_raw(self); + + // SAFETY: `raw` comes from a previous call to `Box::into_raw`. By the safety requirements + // of this function, the value inside the `Box` is in an initialized state. Hence, it is + // safe to reconstruct the `Box` as `Box<T, A>`. + unsafe { Box::from_raw(raw.cast()) } + } + + /// Writes the value and converts to `Box<T, A>`. + pub fn write(mut self, value: T) -> Box<T, A> { + (*self).write(value); + + // SAFETY: We've just initialized `b`'s value. + unsafe { self.assume_init() } + } +} + +impl<T, A> Box<T, A> +where + A: Allocator, +{ + /// Creates a new `Box<T, A>` and initializes its contents with `x`. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + pub fn new(x: T, flags: Flags) -> Result<Self, AllocError> { + let b = Self::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + /// Creates a new `Box<T, A>` with uninitialized contents. + /// + /// New memory is allocated with `A`. The allocation may fail, in which case an error is + /// returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::<u64>::new_uninit(GFP_KERNEL)?; + /// let b = KBox::write(b, 24); + /// + /// assert_eq!(*b, 24_u64); + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>, A>, AllocError> { + let layout = Layout::new::<MaybeUninit<T>>(); + let ptr = A::alloc(layout, flags)?; + + // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, + // which is sufficient in size and alignment for storing a `T`. + Ok(Box(ptr.cast(), PhantomData)) + } + + /// Constructs a new `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then `x` will be + /// pinned in memory and can't be moved. + #[inline] + pub fn pin(x: T, flags: Flags) -> Result<Pin<Box<T, A>>, AllocError> + where + A: 'static, + { + Ok(Self::new(x, flags)?.into()) + } + + /// Forgets the contents (does not run the destructor), but keeps the allocation. + fn forget_contents(this: Self) -> Box<MaybeUninit<T>, A> { + let ptr = Self::into_raw(this); + + // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. + unsafe { Box::from_raw(ptr.cast()) } + } + + /// Drops the contents, but keeps the allocation. + /// + /// # Examples + /// + /// ``` + /// let value = KBox::new([0; 32], GFP_KERNEL)?; + /// assert_eq!(*value, [0; 32]); + /// let value = KBox::drop_contents(value); + /// // Now we can re-use `value`: + /// let value = KBox::write(value, [1; 32]); + /// assert_eq!(*value, [1; 32]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn drop_contents(this: Self) -> Box<MaybeUninit<T>, A> { + let ptr = this.0.as_ptr(); + + // SAFETY: `ptr` is valid, because it came from `this`. After this call we never access the + // value stored in `this` again. + unsafe { core::ptr::drop_in_place(ptr) }; + + Self::forget_contents(this) + } + + /// Moves the `Box`'s value out of the `Box` and consumes the `Box`. + pub fn into_inner(b: Self) -> T { + // SAFETY: By the type invariant `&*b` is valid for `read`. + let value = unsafe { core::ptr::read(&*b) }; + let _ = Self::forget_contents(b); + value + } +} + +impl<T, A> From<Box<T, A>> for Pin<Box<T, A>> +where + T: ?Sized, + A: Allocator, +{ + /// Converts a `Box<T, A>` into a `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then + /// `*b` will be pinned in memory and can't be moved. + /// + /// This moves `b` into `Pin` without moving `*b` or allocating and copying any memory. + fn from(b: Box<T, A>) -> Self { + // SAFETY: The value wrapped inside a `Pin<Box<T, A>>` cannot be moved or replaced as long + // as `T` does not implement `Unpin`. + unsafe { Pin::new_unchecked(b) } + } +} + +impl<T, A> InPlaceWrite<T> for Box<MaybeUninit<T>, A> +where + A: Allocator + 'static, +{ + type Initialized = Box<T, A>; + + fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }) + } + + fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { Box::assume_init(self) }.into()) + } +} + +impl<T, A> InPlaceInit<T> for Box<T, A> +where + A: Allocator + 'static, +{ + type PinnedSelf = Pin<Self>; + + #[inline] + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> + where + E: From<AllocError>, + { + Box::<_, A>::new_uninit(flags)?.write_pin_init(init) + } + + #[inline] + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> + where + E: From<AllocError>, + { + Box::<_, A>::new_uninit(flags)?.write_init(init) + } +} + +impl<T: 'static, A> ForeignOwnable for Box<T, A> +where + A: Allocator, +{ + type Borrowed<'a> = &'a T; + + fn into_foreign(self) -> *const crate::ffi::c_void { + Box::into_raw(self) as _ + } + + unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Box::from_raw(ptr as _) } + } + + unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } +} + +impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>> +where + A: Allocator, +{ + type Borrowed<'a> = Pin<&'a T>; + + fn into_foreign(self) -> *const crate::ffi::c_void { + // SAFETY: We are still treating the box as pinned. + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ + } + + unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } + } + + unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> Pin<&'a T> { + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &*ptr.cast() }; + + // SAFETY: This pointer originates from a `Pin<Box<T>>`. + unsafe { Pin::new_unchecked(r) } + } +} + +impl<T, A> Deref for Box<T, A> +where + T: ?Sized, + A: Allocator, +{ + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_ref() } + } +} + +impl<T, A> DerefMut for Box<T, A> +where + T: ?Sized, + A: Allocator, +{ + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self.0` is always properly aligned, dereferenceable and points to an initialized + // instance of `T`. + unsafe { self.0.as_mut() } + } +} + +impl<T, A> fmt::Debug for Box<T, A> +where + T: ?Sized + fmt::Debug, + A: Allocator, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<T, A> Drop for Box<T, A> +where + T: ?Sized, + A: Allocator, +{ + fn drop(&mut self) { + let layout = Layout::for_value::<T>(self); + + // SAFETY: The pointer in `self.0` is guaranteed to be valid by the type invariant. + unsafe { core::ptr::drop_in_place::<T>(self.deref_mut()) }; + + // SAFETY: + // - `self.0` was previously allocated with `A`. + // - `layout` is equal to the `Layout´ `self.0` was allocated with. + unsafe { A::free(self.0.cast(), layout) }; + } +} diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs new file mode 100644 index 000000000000..ae9d072741ce --- /dev/null +++ b/rust/kernel/alloc/kvec.rs @@ -0,0 +1,913 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Implementation of [`Vec`]. + +use super::{ + allocator::{KVmalloc, Kmalloc, Vmalloc}, + layout::ArrayLayout, + AllocError, Allocator, Box, Flags, +}; +use core::{ + fmt, + marker::PhantomData, + mem::{ManuallyDrop, MaybeUninit}, + ops::Deref, + ops::DerefMut, + ops::Index, + ops::IndexMut, + ptr, + ptr::NonNull, + slice, + slice::SliceIndex, +}; + +/// Create a [`KVec`] containing the arguments. +/// +/// New memory is allocated with `GFP_KERNEL`. +/// +/// # Examples +/// +/// ``` +/// let mut v = kernel::kvec![]; +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(v, [1]); +/// +/// let mut v = kernel::kvec![1; 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 1, 1, 4]); +/// +/// let mut v = kernel::kvec![1, 2, 3]?; +/// v.push(4, GFP_KERNEL)?; +/// assert_eq!(v, [1, 2, 3, 4]); +/// +/// # Ok::<(), Error>(()) +/// ``` +#[macro_export] +macro_rules! kvec { + () => ( + $crate::alloc::KVec::new() + ); + ($elem:expr; $n:expr) => ( + $crate::alloc::KVec::from_elem($elem, $n, GFP_KERNEL) + ); + ($($x:expr),+ $(,)?) => ( + match $crate::alloc::KBox::new_uninit(GFP_KERNEL) { + Ok(b) => Ok($crate::alloc::KVec::from($crate::alloc::KBox::write(b, [$($x),+]))), + Err(e) => Err(e), + } + ); +} + +/// The kernel's [`Vec`] type. +/// +/// A contiguous growable array type with contents allocated with the kernel's allocators (e.g. +/// [`Kmalloc`], [`Vmalloc`] or [`KVmalloc`]), written `Vec<T, A>`. +/// +/// For non-zero-sized values, a [`Vec`] will use the given allocator `A` for its allocation. For +/// the most common allocators the type aliases [`KVec`], [`VVec`] and [`KVVec`] exist. +/// +/// For zero-sized types the [`Vec`]'s pointer must be `dangling_mut::<T>`; no memory is allocated. +/// +/// Generally, [`Vec`] consists of a pointer that represents the vector's backing buffer, the +/// capacity of the vector (the number of elements that currently fit into the vector), its length +/// (the number of elements that are currently stored in the vector) and the `Allocator` type used +/// to allocate (and free) the backing buffer. +/// +/// A [`Vec`] can be deconstructed into and (re-)constructed from its previously named raw parts +/// and manually modified. +/// +/// [`Vec`]'s backing buffer gets, if required, automatically increased (re-allocated) when elements +/// are added to the vector. +/// +/// # Invariants +/// +/// - `self.ptr` is always properly aligned and either points to memory allocated with `A` or, for +/// zero-sized types, is a dangling, well aligned pointer. +/// +/// - `self.len` always represents the exact number of elements stored in the vector. +/// +/// - `self.layout` represents the absolute number of elements that can be stored within the vector +/// without re-allocation. For ZSTs `self.layout`'s capacity is zero. However, it is legal for the +/// backing buffer to be larger than `layout`. +/// +/// - The `Allocator` type `A` of the vector is the exact same `Allocator` type the backing buffer +/// was allocated with (and must be freed with). +pub struct Vec<T, A: Allocator> { + ptr: NonNull<T>, + /// Represents the actual buffer size as `cap` times `size_of::<T>` bytes. + /// + /// Note: This isn't quite the same as `Self::capacity`, which in contrast returns the number of + /// elements we can still store without reallocating. + layout: ArrayLayout<T>, + len: usize, + _p: PhantomData<A>, +} + +/// Type alias for [`Vec`] with a [`Kmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVec<T> = Vec<T, Kmalloc>; + +/// Type alias for [`Vec`] with a [`Vmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = VVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type VVec<T> = Vec<T, Vmalloc>; + +/// Type alias for [`Vec`] with a [`KVmalloc`] allocator. +/// +/// # Examples +/// +/// ``` +/// let mut v = KVVec::new(); +/// v.push(1, GFP_KERNEL)?; +/// assert_eq!(&v, &[1]); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub type KVVec<T> = Vec<T, KVmalloc>; + +// SAFETY: `Vec` is `Send` if `T` is `Send` because `Vec` owns its elements. +unsafe impl<T, A> Send for Vec<T, A> +where + T: Send, + A: Allocator, +{ +} + +// SAFETY: `Vec` is `Sync` if `T` is `Sync` because `Vec` owns its elements. +unsafe impl<T, A> Sync for Vec<T, A> +where + T: Sync, + A: Allocator, +{ +} + +impl<T, A> Vec<T, A> +where + A: Allocator, +{ + #[inline] + const fn is_zst() -> bool { + core::mem::size_of::<T>() == 0 + } + + /// Returns the number of elements that can be stored within the vector without allocating + /// additional memory. + pub fn capacity(&self) -> usize { + if const { Self::is_zst() } { + usize::MAX + } else { + self.layout.len() + } + } + + /// Returns the number of elements stored within the vector. + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Forcefully sets `self.len` to `new_len`. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`Self::capacity`]. + /// - If `new_len` is greater than `self.len`, all elements within the interval + /// [`self.len`,`new_len`) must be initialized. + #[inline] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + self.len = new_len; + } + + /// Returns a slice of the entire vector. + #[inline] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Returns a mutable slice of the entire vector. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a mutable raw pointer to the vector's backing buffer, or, if `T` is a ZST, a + /// dangling raw pointer. + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self.ptr.as_ptr() + } + + /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw + /// pointer. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.ptr.as_ptr() + } + + /// Returns `true` if the vector contains no elements, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1, GFP_KERNEL); + /// assert!(!v.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a new, empty `Vec<T, A>`. + /// + /// This method does not allocate by itself. + #[inline] + pub const fn new() -> Self { + // INVARIANT: Since this is a new, empty `Vec` with no backing memory yet, + // - `ptr` is a properly aligned dangling pointer for type `T`, + // - `layout` is an empty `ArrayLayout` (zero capacity) + // - `len` is zero, since no elements can be or have been stored, + // - `A` is always valid. + Self { + ptr: NonNull::dangling(), + layout: ArrayLayout::empty(), + len: 0, + _p: PhantomData::<A>, + } + } + + /// Returns a slice of `MaybeUninit<T>` for the remaining spare capacity of the vector. + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<T>] { + // SAFETY: + // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is + // guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) } as *mut MaybeUninit<T>; + + // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated + // and valid, but uninitialized. + unsafe { slice::from_raw_parts_mut(ptr, self.capacity() - self.len) } + } + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + self.reserve(1, flags)?; + + // SAFETY: + // - `self.len` is smaller than `self.capacity` and hence, the resulting pointer is + // guaranteed to be part of the same allocated object. + // - `self.len` can not overflow `isize`. + let ptr = unsafe { self.as_mut_ptr().add(self.len) }; + + // SAFETY: + // - `ptr` is properly aligned and valid for writes. + unsafe { core::ptr::write(ptr, v) }; + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because of the previous call to + // `reserve` above. + unsafe { self.set_len(self.len() + 1) }; + Ok(()) + } + + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::<u32>::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError> { + let mut v = Vec::new(); + + v.reserve(capacity, flags)?; + + Ok(v) + } + + /// Creates a `Vec<T, A>` from a pointer, a length and a capacity using the allocator `A`. + /// + /// # Examples + /// + /// ``` + /// let mut v = kernel::kvec![1, 2, 3]?; + /// v.reserve(1, GFP_KERNEL)?; + /// + /// let (mut ptr, mut len, cap) = v.into_raw_parts(); + /// + /// // SAFETY: We've just reserved memory for another element. + /// unsafe { ptr.add(len).write(4) }; + /// len += 1; + /// + /// // SAFETY: We only wrote an additional element at the end of the `KVec`'s buffer and + /// // correspondingly increased the length of the `KVec` by one. Otherwise, we construct it + /// // from the exact same raw parts. + /// let v = unsafe { KVec::from_raw_parts(ptr, len, cap) }; + /// + /// assert_eq!(v, [1, 2, 3, 4]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// If `T` is a ZST: + /// + /// - `ptr` must be a dangling, well aligned pointer. + /// + /// Otherwise: + /// + /// - `ptr` must have been allocated with the allocator `A`. + /// - `ptr` must satisfy or exceed the alignment requirements of `T`. + /// - `ptr` must point to memory with a size of at least `size_of::<T>() * capacity` bytes. + /// - The allocated size in bytes must not be larger than `isize::MAX`. + /// - `length` must be less than or equal to `capacity`. + /// - The first `length` elements must be initialized values of type `T`. + /// + /// It is also valid to create an empty `Vec` passing a dangling pointer for `ptr` and zero for + /// `cap` and `len`. + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + let layout = if Self::is_zst() { + ArrayLayout::empty() + } else { + // SAFETY: By the safety requirements of this function, `capacity * size_of::<T>()` is + // smaller than `isize::MAX`. + unsafe { ArrayLayout::new_unchecked(capacity) } + }; + + // INVARIANT: For ZSTs, we store an empty `ArrayLayout`, all other type invariants are + // covered by the safety requirements of this function. + Self { + // SAFETY: By the safety requirements, `ptr` is either dangling or pointing to a valid + // memory allocation, allocated with `A`. + ptr: unsafe { NonNull::new_unchecked(ptr) }, + layout, + len: length, + _p: PhantomData::<A>, + } + } + + /// Consumes the `Vec<T, A>` and returns its raw components `pointer`, `length` and `capacity`. + /// + /// This will not run the destructor of the contained elements and for non-ZSTs the allocation + /// will stay alive indefinitely. Use [`Vec::from_raw_parts`] to recover the [`Vec`], drop the + /// elements and free the allocation, if any. + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + (ptr, len, capacity) + } + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if Self::is_zst() { + // The capacity is already `usize::MAX` for ZSTs, we can't go higher. + return Err(AllocError); + } + + // We know that `cap <= isize::MAX` because of the type invariants of `Self`. So the + // multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = ArrayLayout::new(new_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `ptr` is valid because it's either `None` or comes from a previous call to + // `A::realloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + A::realloc( + Some(self.ptr.cast()), + layout.into(), + self.layout.into(), + flags, + )? + }; + + // INVARIANT: + // - `layout` is some `ArrayLayout::<T>`, + // - `ptr` has been created by `A::realloc` from `layout`. + self.ptr = ptr.cast(); + self.layout = layout; + + Ok(()) + } +} + +impl<T: Clone, A: Allocator> Vec<T, A> { + /// Extend the vector by `n` clones of `value`. + pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { + if n == 0 { + return Ok(()); + } + + self.reserve(n, flags)?; + + let spare = self.spare_capacity_mut(); + + for item in spare.iter_mut().take(n - 1) { + item.write(value.clone()); + } + + // We can write the last element directly without cloning needlessly. + spare[n - 1].write(value); + + // SAFETY: + // - `self.len() + n < self.capacity()` due to the call to reserve above, + // - the loop and the line above initialized the next `n` elements. + unsafe { self.set_len(self.len() + n) }; + + Ok(()) + } + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> { + self.reserve(other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: + // - `other.len()` spare entries have just been initialized, so it is safe to increase + // the length by the same number. + // - `self.len() + other.len() <= self.capacity()` is guaranteed by the preceding `reserve` + // call. + unsafe { self.set_len(self.len() + other.len()) }; + Ok(()) + } + + /// Create a new `Vec<T, A>` and extend it by `n` clones of `value`. + pub fn from_elem(value: T, n: usize, flags: Flags) -> Result<Self, AllocError> { + let mut v = Self::with_capacity(n, flags)?; + + v.extend_with(n, value, flags)?; + + Ok(v) + } +} + +impl<T, A> Drop for Vec<T, A> +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.as_mut_ptr` is guaranteed to be valid by the type invariant. + unsafe { + ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.as_mut_ptr(), + self.len, + )) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.ptr.cast(), self.layout.into()) }; + } +} + +impl<T, A, const N: usize> From<Box<[T; N], A>> for Vec<T, A> +where + A: Allocator, +{ + fn from(b: Box<[T; N], A>) -> Vec<T, A> { + let len = b.len(); + let ptr = Box::into_raw(b); + + // SAFETY: + // - `b` has been allocated with `A`, + // - `ptr` fulfills the alignment requirements for `T`, + // - `ptr` points to memory with at least a size of `size_of::<T>() * len`, + // - all elements within `b` are initialized values of `T`, + // - `len` does not exceed `isize::MAX`. + unsafe { Vec::from_raw_parts(ptr as _, len, len) } + } +} + +impl<T> Default for KVec<T> { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl<T: fmt::Debug, A: Allocator> fmt::Debug for Vec<T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl<T, A> Deref for Vec<T, A> +where + A: Allocator, +{ + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +impl<T, A> DerefMut for Vec<T, A> +where + A: Allocator, +{ + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + // SAFETY: The memory behind `self.as_ptr()` is guaranteed to contain `self.len` + // initialized elements of type `T`. + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +impl<T: Eq, A> Eq for Vec<T, A> where A: Allocator {} + +impl<T, I: SliceIndex<[T]>, A> Index<I> for Vec<T, A> +where + A: Allocator, +{ + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +impl<T, I: SliceIndex<[T]>, A> IndexMut<I> for Vec<T, A> +where + A: Allocator, +{ + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +macro_rules! impl_slice_eq { + ($([$($vars:tt)*] $lhs:ty, $rhs:ty,)*) => { + $( + impl<T, U, $($vars)*> PartialEq<$rhs> for $lhs + where + T: PartialEq<U>, + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + } + )* + } +} + +impl_slice_eq! { + [A1: Allocator, A2: Allocator] Vec<T, A1>, Vec<U, A2>, + [A: Allocator] Vec<T, A>, &[U], + [A: Allocator] Vec<T, A>, &mut [U], + [A: Allocator] &[T], Vec<U, A>, + [A: Allocator] &mut [T], Vec<U, A>, + [A: Allocator] Vec<T, A>, [U], + [A: Allocator] [T], Vec<U, A>, + [A: Allocator, const N: usize] Vec<T, A>, [U; N], + [A: Allocator, const N: usize] Vec<T, A>, &[U; N], +} + +impl<'a, T, A> IntoIterator for &'a Vec<T, A> +where + A: Allocator, +{ + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec<T, A> +where + A: Allocator, +{ + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector. +/// +/// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the +/// [`IntoIterator`] trait). +/// +/// # Examples +/// +/// ``` +/// let v = kernel::kvec![0, 1, 2]?; +/// let iter = v.into_iter(); +/// +/// # Ok::<(), Error>(()) +/// ``` +pub struct IntoIter<T, A: Allocator> { + ptr: *mut T, + buf: NonNull<T>, + len: usize, + layout: ArrayLayout<T>, + _p: PhantomData<A>, +} + +impl<T, A> IntoIter<T, A> +where + A: Allocator, +{ + fn into_raw_parts(self) -> (*mut T, NonNull<T>, usize, usize) { + let me = ManuallyDrop::new(self); + let ptr = me.ptr; + let buf = me.buf; + let len = me.len; + let cap = me.layout.len(); + (ptr, buf, len, cap) + } + + /// Same as `Iterator::collect` but specialized for `Vec`'s `IntoIter`. + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// + /// let v = it.collect(GFP_KERNEL); + /// assert_eq!(v, [2, 3]); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Implementation details + /// + /// Currently, we can't implement `FromIterator`. There are a couple of issues with this trait + /// in the kernel, namely: + /// + /// - Rust's specialization feature is unstable. This prevents us to optimize for the special + /// case where `I::IntoIter` equals `Vec`'s `IntoIter` type. + /// - We also can't use `I::IntoIter`'s type ID either to work around this, since `FromIterator` + /// doesn't require this type to be `'static`. + /// - `FromIterator::from_iter` does return `Self` instead of `Result<Self, AllocError>`, hence + /// we can't properly handle allocation failures. + /// - Neither `Iterator::collect` nor `FromIterator::from_iter` can handle additional allocation + /// flags. + /// + /// Instead, provide `IntoIter::collect`, such that we can at least convert a `IntoIter` into a + /// `Vec` again. + /// + /// Note that `IntoIter::collect` doesn't require `Flags`, since it re-uses the existing backing + /// buffer. However, this backing buffer may be shrunk to the actual count of elements. + pub fn collect(self, flags: Flags) -> Vec<T, A> { + let old_layout = self.layout; + let (mut ptr, buf, len, mut cap) = self.into_raw_parts(); + let has_advanced = ptr != buf.as_ptr(); + + if has_advanced { + // Copy the contents we have advanced to at the beginning of the buffer. + // + // SAFETY: + // - `ptr` is valid for reads of `len * size_of::<T>()` bytes, + // - `buf.as_ptr()` is valid for writes of `len * size_of::<T>()` bytes, + // - `ptr` and `buf.as_ptr()` are not be subject to aliasing restrictions relative to + // each other, + // - both `ptr` and `buf.ptr()` are properly aligned. + unsafe { ptr::copy(ptr, buf.as_ptr(), len) }; + ptr = buf.as_ptr(); + + // SAFETY: `len` is guaranteed to be smaller than `self.layout.len()`. + let layout = unsafe { ArrayLayout::<T>::new_unchecked(len) }; + + // SAFETY: `buf` points to the start of the backing buffer and `len` is guaranteed to be + // smaller than `cap`. Depending on `alloc` this operation may shrink the buffer or leaves + // it as it is. + ptr = match unsafe { + A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags) + } { + // If we fail to shrink, which likely can't even happen, continue with the existing + // buffer. + Err(_) => ptr, + Ok(ptr) => { + cap = len; + ptr.as_ptr().cast() + } + }; + } + + // SAFETY: If the iterator has been advanced, the advanced elements have been copied to + // the beginning of the buffer and `len` has been adjusted accordingly. + // + // - `ptr` is guaranteed to point to the start of the backing buffer. + // - `cap` is either the original capacity or, after shrinking the buffer, equal to `len`. + // - `alloc` is guaranteed to be unchanged since `into_iter` has been called on the original + // `Vec`. + unsafe { Vec::from_raw_parts(ptr, len, cap) } + } +} + +impl<T, A> Iterator for IntoIter<T, A> +where + A: Allocator, +{ + type Item = T; + + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2, 3]?; + /// let mut it = v.into_iter(); + /// + /// assert_eq!(it.next(), Some(1)); + /// assert_eq!(it.next(), Some(2)); + /// assert_eq!(it.next(), Some(3)); + /// assert_eq!(it.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn next(&mut self) -> Option<T> { + if self.len == 0 { + return None; + } + + let current = self.ptr; + + // SAFETY: We can't overflow; decreasing `self.len` by one every time we advance `self.ptr` + // by one guarantees that. + unsafe { self.ptr = self.ptr.add(1) }; + + self.len -= 1; + + // SAFETY: `current` is guaranteed to point at a valid element within the buffer. + Some(unsafe { current.read() }) + } + + /// # Examples + /// + /// ``` + /// let v: KVec<u32> = kernel::kvec![1, 2, 3]?; + /// let mut iter = v.into_iter(); + /// let size = iter.size_hint().0; + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 1); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 2); + /// + /// iter.next(); + /// assert_eq!(iter.size_hint().0, size - 3); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len, Some(self.len)) + } +} + +impl<T, A> Drop for IntoIter<T, A> +where + A: Allocator, +{ + fn drop(&mut self) { + // SAFETY: `self.ptr` is guaranteed to be valid by the type invariant. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr, self.len)) }; + + // SAFETY: + // - `self.buf` was previously allocated with `A`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { A::free(self.buf.cast(), self.layout.into()) }; + } +} + +impl<T, A> IntoIterator for Vec<T, A> +where + A: Allocator, +{ + type Item = T; + type IntoIter = IntoIter<T, A>; + + /// Consumes the `Vec<T, A>` and creates an `Iterator`, which moves each value out of the + /// vector (from start to end). + /// + /// # Examples + /// + /// ``` + /// let v = kernel::kvec![1, 2]?; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option<u32> = v_iter.next(); + /// + /// assert_eq!(first_element, Some(1)); + /// assert_eq!(v_iter.next(), Some(2)); + /// assert_eq!(v_iter.next(), None); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// let v = kernel::kvec![]; + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option<u32> = v_iter.next(); + /// + /// assert_eq!(first_element, None); + /// + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + fn into_iter(self) -> Self::IntoIter { + let buf = self.ptr; + let layout = self.layout; + let (ptr, len, _) = self.into_raw_parts(); + + IntoIter { + ptr, + buf, + len, + layout, + _p: PhantomData::<A>, + } + } +} diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs new file mode 100644 index 000000000000..4b3cd7fdc816 --- /dev/null +++ b/rust/kernel/alloc/layout.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory layout. +//! +//! Custom layout types extending or improving [`Layout`]. + +use core::{alloc::Layout, marker::PhantomData}; + +/// Error when constructing an [`ArrayLayout`]. +pub struct LayoutError; + +/// A layout for an array `[T; n]`. +/// +/// # Invariants +/// +/// - `len * size_of::<T>() <= isize::MAX`. +pub struct ArrayLayout<T> { + len: usize, + _phantom: PhantomData<fn() -> T>, +} + +impl<T> Clone for ArrayLayout<T> { + fn clone(&self) -> Self { + *self + } +} +impl<T> Copy for ArrayLayout<T> {} + +const ISIZE_MAX: usize = isize::MAX as usize; + +impl<T> ArrayLayout<T> { + /// Creates a new layout for `[T; 0]`. + pub const fn empty() -> Self { + // INVARIANT: `0 * size_of::<T>() <= isize::MAX`. + Self { + len: 0, + _phantom: PhantomData, + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Errors + /// + /// When `len * size_of::<T>()` overflows or when `len * size_of::<T>() > isize::MAX`. + pub const fn new(len: usize) -> Result<Self, LayoutError> { + match len.checked_mul(core::mem::size_of::<T>()) { + Some(size) if size <= ISIZE_MAX => { + // INVARIANT: We checked above that `len * size_of::<T>() <= isize::MAX`. + Ok(Self { + len, + _phantom: PhantomData, + }) + } + _ => Err(LayoutError), + } + } + + /// Creates a new layout for `[T; len]`. + /// + /// # Safety + /// + /// `len` must be a value, for which `len * size_of::<T>() <= isize::MAX` is true. + pub unsafe fn new_unchecked(len: usize) -> Self { + // INVARIANT: By the safety requirements of this function + // `len * size_of::<T>() <= isize::MAX`. + Self { + len, + _phantom: PhantomData, + } + } + + /// Returns the number of array elements represented by this layout. + pub const fn len(&self) -> usize { + self.len + } + + /// Returns `true` when no array elements are represented by this layout. + pub const fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl<T> From<ArrayLayout<T>> for Layout { + fn from(value: ArrayLayout<T>) -> Self { + let res = Layout::array::<T>(value.len); + // SAFETY: By the type invariant of `ArrayLayout` we have + // `len * size_of::<T>() <= isize::MAX` and thus the result must be `Ok`. + unsafe { res.unwrap_unchecked() } + } +} diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs deleted file mode 100644 index 1297a4be32e8..000000000000 --- a/rust/kernel/alloc/vec_ext.rs +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Extensions to [`Vec`] for fallible allocations. - -use super::{AllocError, Flags}; -use alloc::vec::Vec; - -/// Extensions to [`Vec`]. -pub trait VecExt<T>: Sized { - /// Creates a new [`Vec`] instance with at least the given capacity. - /// - /// # Examples - /// - /// ``` - /// let v = Vec::<u32>::with_capacity(20, GFP_KERNEL)?; - /// - /// assert!(v.capacity() >= 20); - /// # Ok::<(), Error>(()) - /// ``` - fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError>; - - /// Appends an element to the back of the [`Vec`] instance. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// assert_eq!(&v, &[1]); - /// - /// v.push(2, GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 2]); - /// # Ok::<(), Error>(()) - /// ``` - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>; - - /// Pushes clones of the elements of slice into the [`Vec`] instance. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// - /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 20, 30, 40]); - /// - /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; - /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); - /// # Ok::<(), Error>(()) - /// ``` - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> - where - T: Clone; - - /// Ensures that the capacity exceeds the length by at least `additional` elements. - /// - /// # Examples - /// - /// ``` - /// let mut v = Vec::new(); - /// v.push(1, GFP_KERNEL)?; - /// - /// v.reserve(10, GFP_KERNEL)?; - /// let cap = v.capacity(); - /// assert!(cap >= 10); - /// - /// v.reserve(10, GFP_KERNEL)?; - /// let new_cap = v.capacity(); - /// assert_eq!(new_cap, cap); - /// - /// # Ok::<(), Error>(()) - /// ``` - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>; -} - -impl<T> VecExt<T> for Vec<T> { - fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError> { - let mut v = Vec::new(); - <Self as VecExt<_>>::reserve(&mut v, capacity, flags)?; - Ok(v) - } - - fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { - <Self as VecExt<_>>::reserve(self, 1, flags)?; - let s = self.spare_capacity_mut(); - s[0].write(v); - - // SAFETY: We just initialised the first spare entry, so it is safe to increase the length - // by 1. We also know that the new length is <= capacity because of the previous call to - // `reserve` above. - unsafe { self.set_len(self.len() + 1) }; - Ok(()) - } - - fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> - where - T: Clone, - { - <Self as VecExt<_>>::reserve(self, other.len(), flags)?; - for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { - slot.write(item.clone()); - } - - // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase - // the length by the same amount. We also know that the new length is <= capacity because - // of the previous call to `reserve` above. - unsafe { self.set_len(self.len() + other.len()) }; - Ok(()) - } - - #[cfg(any(test, testlib))] - fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> { - Vec::reserve(self, additional); - Ok(()) - } - - #[cfg(not(any(test, testlib)))] - fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { - let len = self.len(); - let cap = self.capacity(); - - if cap - len >= additional { - return Ok(()); - } - - if core::mem::size_of::<T>() == 0 { - // The capacity is already `usize::MAX` for SZTs, we can't go higher. - return Err(AllocError); - } - - // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size - // is greater than `isize::MAX`. So the multiplication by two won't overflow. - let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); - let layout = core::alloc::Layout::array::<T>(new_cap).map_err(|_| AllocError)?; - - let (old_ptr, len, cap) = destructure(self); - - // We need to make sure that `ptr` is either NULL or comes from a previous call to - // `krealloc_aligned`. A `Vec<T>`'s `ptr` value is not guaranteed to be NULL and might be - // dangling after being created with `Vec::new`. Instead, we can rely on `Vec<T>`'s capacity - // to be zero if no memory has been allocated yet. - let ptr = if cap == 0 { - core::ptr::null_mut() - } else { - old_ptr - }; - - // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to - // `krealloc_aligned`. We also verified that the type is not a ZST. - let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) }; - if new_ptr.is_null() { - // SAFETY: We are just rebuilding the existing `Vec` with no changes. - unsafe { rebuild(self, old_ptr, len, cap) }; - Err(AllocError) - } else { - // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap - // is greater than `cap`, so it continues to be >= `len`. - unsafe { rebuild(self, new_ptr.cast::<T>(), len, new_cap) }; - Ok(()) - } - } -} - -#[cfg(not(any(test, testlib)))] -fn destructure<T>(v: &mut Vec<T>) -> (*mut T, usize, usize) { - let mut tmp = Vec::new(); - core::mem::swap(&mut tmp, v); - let mut tmp = core::mem::ManuallyDrop::new(tmp); - let len = tmp.len(); - let cap = tmp.capacity(); - (tmp.as_mut_ptr(), len, cap) -} - -/// Rebuilds a `Vec` from a pointer, length, and capacity. -/// -/// # Safety -/// -/// The same as [`Vec::from_raw_parts`]. -#[cfg(not(any(test, testlib)))] -unsafe fn rebuild<T>(v: &mut Vec<T>, ptr: *mut T, len: usize, cap: usize) { - // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`. - let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) }; - core::mem::swap(&mut tmp, v); -} diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index 9ba7fdfeb4b2..c8646d0d9866 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -131,7 +131,7 @@ impl<T: Operations> OperationsVTable<T> { unsafe extern "C" fn poll_callback( _hctx: *mut bindings::blk_mq_hw_ctx, _iob: *mut bindings::io_comp_batch, - ) -> core::ffi::c_int { + ) -> crate::ffi::c_int { T::poll().into() } @@ -145,9 +145,9 @@ impl<T: Operations> OperationsVTable<T> { /// for the same context. unsafe extern "C" fn init_hctx_callback( _hctx: *mut bindings::blk_mq_hw_ctx, - _tagset_data: *mut core::ffi::c_void, - _hctx_idx: core::ffi::c_uint, - ) -> core::ffi::c_int { + _tagset_data: *mut crate::ffi::c_void, + _hctx_idx: crate::ffi::c_uint, + ) -> crate::ffi::c_int { from_result(|| Ok(0)) } @@ -159,7 +159,7 @@ impl<T: Operations> OperationsVTable<T> { /// This function may only be called by blk-mq C infrastructure. unsafe extern "C" fn exit_hctx_callback( _hctx: *mut bindings::blk_mq_hw_ctx, - _hctx_idx: core::ffi::c_uint, + _hctx_idx: crate::ffi::c_uint, ) { } @@ -176,9 +176,9 @@ impl<T: Operations> OperationsVTable<T> { unsafe extern "C" fn init_request_callback( _set: *mut bindings::blk_mq_tag_set, rq: *mut bindings::request, - _hctx_idx: core::ffi::c_uint, - _numa_node: core::ffi::c_uint, - ) -> core::ffi::c_int { + _hctx_idx: crate::ffi::c_uint, + _numa_node: crate::ffi::c_uint, + ) -> crate::ffi::c_int { from_result(|| { // SAFETY: By the safety requirements of this function, `rq` points // to a valid allocation. @@ -203,7 +203,7 @@ impl<T: Operations> OperationsVTable<T> { unsafe extern "C" fn exit_request_callback( _set: *mut bindings::blk_mq_tag_set, rq: *mut bindings::request, - _hctx_idx: core::ffi::c_uint, + _hctx_idx: crate::ffi::c_uint, ) { // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory // for the request data. diff --git a/rust/kernel/block/mq/raw_writer.rs b/rust/kernel/block/mq/raw_writer.rs index 9222465d670b..7e2159e4f6a6 100644 --- a/rust/kernel/block/mq/raw_writer.rs +++ b/rust/kernel/block/mq/raw_writer.rs @@ -25,7 +25,7 @@ impl<'a> RawWriter<'a> { } pub(crate) fn from_array<const N: usize>( - a: &'a mut [core::ffi::c_char; N], + a: &'a mut [crate::ffi::c_char; N], ) -> Result<RawWriter<'a>> { Self::new( // SAFETY: the buffer of `a` is valid for read and write as `u8` for diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index a0e22827f3f4..7943f43b9575 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -16,50 +16,55 @@ use core::{ sync::atomic::{AtomicU64, Ordering}, }; -/// A wrapper around a blk-mq `struct request`. This represents an IO request. +/// A wrapper around a blk-mq [`struct request`]. This represents an IO request. /// /// # Implementation details /// /// There are four states for a request that the Rust bindings care about: /// -/// A) Request is owned by block layer (refcount 0) -/// B) Request is owned by driver but with zero `ARef`s in existence -/// (refcount 1) -/// C) Request is owned by driver with exactly one `ARef` in existence -/// (refcount 2) -/// D) Request is owned by driver with more than one `ARef` in existence -/// (refcount > 2) +/// 1. Request is owned by block layer (refcount 0). +/// 2. Request is owned by driver but with zero [`ARef`]s in existence +/// (refcount 1). +/// 3. Request is owned by driver with exactly one [`ARef`] in existence +/// (refcount 2). +/// 4. Request is owned by driver with more than one [`ARef`] in existence +/// (refcount > 2). /// /// -/// We need to track A and B to ensure we fail tag to request conversions for +/// We need to track 1 and 2 to ensure we fail tag to request conversions for /// requests that are not owned by the driver. /// -/// We need to track C and D to ensure that it is safe to end the request and hand +/// We need to track 3 and 4 to ensure that it is safe to end the request and hand /// back ownership to the block layer. /// /// The states are tracked through the private `refcount` field of /// `RequestDataWrapper`. This structure lives in the private data area of the C -/// `struct request`. +/// [`struct request`]. /// /// # Invariants /// -/// * `self.0` is a valid `struct request` created by the C portion of the kernel. +/// * `self.0` is a valid [`struct request`] created by the C portion of the +/// kernel. /// * The private data area associated with this request must be an initialized /// and valid `RequestDataWrapper<T>`. /// * `self` is reference counted by atomic modification of -/// self.wrapper_ref().refcount(). +/// `self.wrapper_ref().refcount()`. +/// +/// [`struct request`]: srctree/include/linux/blk-mq.h /// #[repr(transparent)] pub struct Request<T: Operations>(Opaque<bindings::request>, PhantomData<T>); impl<T: Operations> Request<T> { - /// Create an `ARef<Request>` from a `struct request` pointer. + /// Create an [`ARef<Request>`] from a [`struct request`] pointer. /// /// # Safety /// /// * The caller must own a refcount on `ptr` that is transferred to the - /// returned `ARef`. - /// * The type invariants for `Request` must hold for the pointee of `ptr`. + /// returned [`ARef`]. + /// * The type invariants for [`Request`] must hold for the pointee of `ptr`. + /// + /// [`struct request`]: srctree/include/linux/blk-mq.h pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef<Self> { // INVARIANT: By the safety requirements of this function, invariants are upheld. // SAFETY: By the safety requirement of this function, we own a @@ -84,12 +89,14 @@ impl<T: Operations> Request<T> { } /// Try to take exclusive ownership of `this` by dropping the refcount to 0. - /// This fails if `this` is not the only `ARef` pointing to the underlying - /// `Request`. + /// This fails if `this` is not the only [`ARef`] pointing to the underlying + /// [`Request`]. /// - /// If the operation is successful, `Ok` is returned with a pointer to the - /// C `struct request`. If the operation fails, `this` is returned in the - /// `Err` variant. + /// If the operation is successful, [`Ok`] is returned with a pointer to the + /// C [`struct request`]. If the operation fails, `this` is returned in the + /// [`Err`] variant. + /// + /// [`struct request`]: srctree/include/linux/blk-mq.h fn try_set_end(this: ARef<Self>) -> Result<*mut bindings::request, ARef<Self>> { // We can race with `TagSet::tag_to_rq` if let Err(_old) = this.wrapper_ref().refcount().compare_exchange( @@ -109,7 +116,7 @@ impl<T: Operations> Request<T> { /// Notify the block layer that the request has been completed without errors. /// - /// This function will return `Err` if `this` is not the only `ARef` + /// This function will return [`Err`] if `this` is not the only [`ARef`] /// referencing the request. pub fn end_ok(this: ARef<Self>) -> Result<(), ARef<Self>> { let request_ptr = Self::try_set_end(this)?; @@ -123,13 +130,13 @@ impl<T: Operations> Request<T> { Ok(()) } - /// Return a pointer to the `RequestDataWrapper` stored in the private area + /// Return a pointer to the [`RequestDataWrapper`] stored in the private area /// of the request structure. /// /// # Safety /// /// - `this` must point to a valid allocation of size at least size of - /// `Self` plus size of `RequestDataWrapper`. + /// [`Self`] plus size of [`RequestDataWrapper`]. pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull<RequestDataWrapper> { let request_ptr = this.cast::<bindings::request>(); // SAFETY: By safety requirements for this function, `this` is a @@ -141,7 +148,7 @@ impl<T: Operations> Request<T> { unsafe { NonNull::new_unchecked(wrapper_ptr) } } - /// Return a reference to the `RequestDataWrapper` stored in the private + /// Return a reference to the [`RequestDataWrapper`] stored in the private /// area of the request structure. pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { // SAFETY: By type invariant, `self.0` is a valid allocation. Further, @@ -152,13 +159,15 @@ impl<T: Operations> Request<T> { } } -/// A wrapper around data stored in the private area of the C `struct request`. +/// A wrapper around data stored in the private area of the C [`struct request`]. +/// +/// [`struct request`]: srctree/include/linux/blk-mq.h pub(crate) struct RequestDataWrapper { /// The Rust request refcount has the following states: /// /// - 0: The request is owned by C block layer. - /// - 1: The request is owned by Rust abstractions but there are no ARef references to it. - /// - 2+: There are `ARef` references to the request. + /// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. + /// - 2+: There are [`ARef`] references to the request. refcount: AtomicU64, } @@ -204,7 +213,7 @@ fn atomic_relaxed_op_return(target: &AtomicU64, op: impl Fn(u64) -> u64) -> u64 } /// Store the result of `op(target.load)` in `target` if `target.load() != -/// pred`, returning true if the target was updated. +/// pred`, returning [`true`] if the target was updated. fn atomic_relaxed_op_unless(target: &AtomicU64, op: impl Fn(u64) -> u64, pred: u64) -> bool { target .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |x| { diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index f9a1ca655a35..d7f175a05d99 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -53,7 +53,7 @@ impl<T: Operations> TagSet<T> { queue_depth: num_tags, cmd_size, flags: bindings::BLK_MQ_F_SHOULD_MERGE, - driver_data: core::ptr::null_mut::<core::ffi::c_void>(), + driver_data: core::ptr::null_mut::<crate::ffi::c_void>(), nr_maps: num_maps, ..tag_set } diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 6f1587a2524e..52c502432447 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -6,9 +6,10 @@ use crate::{alloc::AllocError, str::CStr}; -use alloc::alloc::LayoutError; +use core::alloc::LayoutError; use core::fmt; +use core::num::NonZeroI32; use core::num::TryFromIntError; use core::str::Utf8Error; @@ -20,7 +21,11 @@ pub mod code { $( #[doc = $doc] )* - pub const $err: super::Error = super::Error(-(crate::bindings::$err as i32)); + pub const $err: super::Error = + match super::Error::try_from_errno(-(crate::bindings::$err as i32)) { + Some(err) => err, + None => panic!("Invalid errno in `declare_err!`"), + }; }; } @@ -88,14 +93,14 @@ pub mod code { /// /// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`). #[derive(Clone, Copy, PartialEq, Eq)] -pub struct Error(core::ffi::c_int); +pub struct Error(NonZeroI32); impl Error { /// Creates an [`Error`] from a kernel error code. /// /// It is a bug to pass an out-of-range `errno`. `EINVAL` would /// be returned in such a case. - pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error { + pub fn from_errno(errno: crate::ffi::c_int) -> Error { if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { // TODO: Make it a `WARN_ONCE` once available. crate::pr_warn!( @@ -107,7 +112,20 @@ impl Error { // INVARIANT: The check above ensures the type invariant // will hold. - Error(errno) + // SAFETY: `errno` is checked above to be in a valid range. + unsafe { Error::from_errno_unchecked(errno) } + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// Returns [`None`] if `errno` is out-of-range. + const fn try_from_errno(errno: crate::ffi::c_int) -> Option<Error> { + if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { + return None; + } + + // SAFETY: `errno` is checked above to be in a valid range. + Some(unsafe { Error::from_errno_unchecked(errno) }) } /// Creates an [`Error`] from a kernel error code. @@ -115,38 +133,38 @@ impl Error { /// # Safety /// /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). - unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error { + const unsafe fn from_errno_unchecked(errno: crate::ffi::c_int) -> Error { // INVARIANT: The contract ensures the type invariant // will hold. - Error(errno) + // SAFETY: The caller guarantees `errno` is non-zero. + Error(unsafe { NonZeroI32::new_unchecked(errno) }) } /// Returns the kernel error code. - pub fn to_errno(self) -> core::ffi::c_int { - self.0 + pub fn to_errno(self) -> crate::ffi::c_int { + self.0.get() } #[cfg(CONFIG_BLOCK)] pub(crate) fn to_blk_status(self) -> bindings::blk_status_t { // SAFETY: `self.0` is a valid error due to its invariant. - unsafe { bindings::errno_to_blk_status(self.0) } + unsafe { bindings::errno_to_blk_status(self.0.get()) } } /// Returns the error encoded as a pointer. - #[allow(dead_code)] - pub(crate) fn to_ptr<T>(self) -> *mut T { + pub fn to_ptr<T>(self) -> *mut T { #[cfg_attr(target_pointer_width = "32", allow(clippy::useless_conversion))] // SAFETY: `self.0` is a valid error due to its invariant. unsafe { - bindings::ERR_PTR(self.0.into()) as *mut _ + bindings::ERR_PTR(self.0.get().into()) as *mut _ } } /// Returns a string representing the error, if one exists. - #[cfg(not(testlib))] + #[cfg(not(any(test, testlib)))] pub fn name(&self) -> Option<&'static CStr> { // SAFETY: Just an FFI call, there are no extra safety requirements. - let ptr = unsafe { bindings::errname(-self.0) }; + let ptr = unsafe { bindings::errname(-self.0.get()) }; if ptr.is_null() { None } else { @@ -160,7 +178,7 @@ impl Error { /// When `testlib` is configured, this always returns `None` to avoid the dependency on a /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still /// run in userspace. - #[cfg(testlib)] + #[cfg(any(test, testlib))] pub fn name(&self) -> Option<&'static CStr> { None } @@ -171,9 +189,11 @@ impl fmt::Debug for Error { match self.name() { // Print out number if no name can be found. None => f.debug_tuple("Error").field(&-self.0).finish(), - // SAFETY: These strings are ASCII-only. Some(name) => f - .debug_tuple(unsafe { core::str::from_utf8_unchecked(name) }) + .debug_tuple( + // SAFETY: These strings are ASCII-only. + unsafe { core::str::from_utf8_unchecked(name) }, + ) .finish(), } } @@ -239,7 +259,7 @@ pub type Result<T = (), E = Error> = core::result::Result<T, E>; /// Converts an integer as returned by a C kernel function to an error if it's negative, and /// `Ok(())` otherwise. -pub fn to_result(err: core::ffi::c_int) -> Result { +pub fn to_result(err: crate::ffi::c_int) -> Result { if err < 0 { Err(Error::from_errno(err)) } else { @@ -262,21 +282,21 @@ pub fn to_result(err: core::ffi::c_int) -> Result { /// fn devm_platform_ioremap_resource( /// pdev: &mut PlatformDevice, /// index: u32, -/// ) -> Result<*mut core::ffi::c_void> { +/// ) -> Result<*mut kernel::ffi::c_void> { /// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements /// // on `index`. /// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) /// } /// ``` -// TODO: Remove `dead_code` marker once an in-kernel client is available. -#[allow(dead_code)] -pub(crate) fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { - // CAST: Casting a pointer to `*const core::ffi::c_void` is always valid. - let const_ptr: *const core::ffi::c_void = ptr.cast(); +pub fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { + // CAST: Casting a pointer to `*const crate::ffi::c_void` is always valid. + let const_ptr: *const crate::ffi::c_void = ptr.cast(); // SAFETY: The FFI function does not deref the pointer. if unsafe { bindings::IS_ERR(const_ptr) } { // SAFETY: The FFI function does not deref the pointer. let err = unsafe { bindings::PTR_ERR(const_ptr) }; + + #[allow(clippy::unnecessary_cast)] // CAST: If `IS_ERR()` returns `true`, // then `PTR_ERR()` is guaranteed to return a // negative value greater-or-equal to `-bindings::MAX_ERRNO`, @@ -286,8 +306,7 @@ pub(crate) fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { // // SAFETY: `IS_ERR()` ensures `err` is a // negative value greater-or-equal to `-bindings::MAX_ERRNO`. - #[allow(clippy::unnecessary_cast)] - return Err(unsafe { Error::from_errno_unchecked(err as core::ffi::c_int) }); + return Err(unsafe { Error::from_errno_unchecked(err as crate::ffi::c_int) }); } Ok(ptr) } @@ -307,7 +326,7 @@ pub(crate) fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { /// # use kernel::bindings; /// unsafe extern "C" fn probe_callback( /// pdev: *mut bindings::platform_device, -/// ) -> core::ffi::c_int { +/// ) -> kernel::ffi::c_int { /// from_result(|| { /// let ptr = devm_alloc(pdev)?; /// bindings::platform_set_drvdata(pdev, ptr); @@ -315,9 +334,7 @@ pub(crate) fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { /// }) /// } /// ``` -// TODO: Remove `dead_code` marker once an in-kernel client is available. -#[allow(dead_code)] -pub(crate) fn from_result<T, F>(f: F) -> T +pub fn from_result<T, F>(f: F) -> T where T: From<i16>, F: FnOnce() -> Result<T>, diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index a17ac8762d8f..347049df556b 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -13,7 +13,7 @@ //! To initialize a `struct` with an in-place constructor you will need two things: //! - an in-place constructor, //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], -//! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]). +//! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]). //! //! To get an in-place constructor there are generally three options: //! - directly creating an in-place constructor using the [`pin_init!`] macro, @@ -35,7 +35,7 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![allow(clippy::disallowed_names)] +//! # #![expect(clippy::disallowed_names)] //! use kernel::sync::{new_mutex, Mutex}; //! # use core::pin::Pin; //! #[pin_data] @@ -55,7 +55,7 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![allow(clippy::disallowed_names)] +//! # #![expect(clippy::disallowed_names)] //! # use kernel::sync::{new_mutex, Mutex}; //! # use core::pin::Pin; //! # #[pin_data] @@ -68,7 +68,7 @@ //! # a <- new_mutex!(42, "Foo::a"), //! # b: 24, //! # }); -//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo, GFP_KERNEL); +//! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL); //! ``` //! //! For more information see the [`pin_init!`] macro. @@ -87,20 +87,19 @@ //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: //! //! ```rust -//! # #![allow(clippy::disallowed_names)] //! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init}; //! #[pin_data] //! struct DriverData { //! #[pin] //! status: Mutex<i32>, -//! buffer: Box<[u8; 1_000_000]>, +//! buffer: KBox<[u8; 1_000_000]>, //! } //! //! impl DriverData { //! fn new() -> impl PinInit<Self, Error> { //! try_pin_init!(Self { //! status <- new_mutex!(0, "DriverData::status"), -//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?, +//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?, //! }) //! } //! } @@ -121,11 +120,12 @@ //! `slot` gets called. //! //! ```rust -//! # #![allow(unreachable_pub, clippy::disallowed_names)] +//! # #![expect(unreachable_pub, clippy::disallowed_names)] //! use kernel::{init, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { -//! # #![allow(non_camel_case_types)] +//! # #![expect(non_camel_case_types)] +//! # #![expect(clippy::missing_safety_doc)] //! # pub struct foo; //! # pub unsafe fn init_foo(_ptr: *mut foo) {} //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} @@ -133,7 +133,7 @@ //! # } //! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. //! # trait FromErrno { -//! # fn from_errno(errno: core::ffi::c_int) -> Error { +//! # fn from_errno(errno: kernel::ffi::c_int) -> Error { //! # // Dummy error that can be constructed outside the `kernel` crate. //! # Error::from(core::fmt::Error) //! # } @@ -211,13 +211,12 @@ //! [`pin_init!`]: crate::pin_init! use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags}, + alloc::{AllocError, Flags, KBox}, error::{self, Error}, sync::Arc, sync::UniqueArc, types::{Opaque, ScopeGuard}, }; -use alloc::boxed::Box; use core::{ cell::UnsafeCell, convert::Infallible, @@ -238,7 +237,7 @@ pub mod macros; /// # Examples /// /// ```rust -/// # #![allow(clippy::disallowed_names)] +/// # #![expect(clippy::disallowed_names)] /// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; /// # use core::pin::Pin; /// #[pin_data] @@ -290,7 +289,7 @@ macro_rules! stack_pin_init { /// # Examples /// /// ```rust,ignore -/// # #![allow(clippy::disallowed_names)] +/// # #![expect(clippy::disallowed_names)] /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; /// # use macros::pin_data; /// # use core::{alloc::AllocError, pin::Pin}; @@ -298,7 +297,7 @@ macro_rules! stack_pin_init { /// struct Foo { /// #[pin] /// a: Mutex<usize>, -/// b: Box<Bar>, +/// b: KBox<Bar>, /// } /// /// struct Bar { @@ -307,7 +306,7 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::new(Bar { +/// b: KBox::new(Bar { /// x: 64, /// }, GFP_KERNEL)?, /// })); @@ -316,7 +315,7 @@ macro_rules! stack_pin_init { /// ``` /// /// ```rust,ignore -/// # #![allow(clippy::disallowed_names)] +/// # #![expect(clippy::disallowed_names)] /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; /// # use macros::pin_data; /// # use core::{alloc::AllocError, pin::Pin}; @@ -324,7 +323,7 @@ macro_rules! stack_pin_init { /// struct Foo { /// #[pin] /// a: Mutex<usize>, -/// b: Box<Bar>, +/// b: KBox<Bar>, /// } /// /// struct Bar { @@ -333,7 +332,7 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::new(Bar { +/// b: KBox::new(Bar { /// x: 64, /// }, GFP_KERNEL)?, /// })); @@ -368,7 +367,6 @@ macro_rules! stack_try_pin_init { /// The syntax is almost identical to that of a normal `struct` initializer: /// /// ```rust -/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// #[pin_data] @@ -392,7 +390,7 @@ macro_rules! stack_try_pin_init { /// }, /// }); /// # initializer } -/// # Box::pin_init(demo(), GFP_KERNEL).unwrap(); +/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap(); /// ``` /// /// Arbitrary Rust expressions can be used to set the value of a variable. @@ -413,7 +411,6 @@ macro_rules! stack_try_pin_init { /// To create an initializer function, simply declare it like this: /// /// ```rust -/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -440,7 +437,7 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![allow(clippy::disallowed_names)] +/// # #![expect(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -462,13 +459,12 @@ macro_rules! stack_try_pin_init { /// # }) /// # } /// # } -/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL); +/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL); /// ``` /// /// They can also easily embed it into their own `struct`s: /// /// ```rust -/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -541,6 +537,7 @@ macro_rules! stack_try_pin_init { /// } /// pin_init!(&this in Buf { /// buf: [0; 64], +/// // SAFETY: TODO. /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, /// pin: PhantomPinned, /// }); @@ -590,11 +587,10 @@ macro_rules! pin_init { /// # Examples /// /// ```rust -/// # #![feature(new_uninit)] /// use kernel::{init::{self, PinInit}, error::Error}; /// #[pin_data] /// struct BigBuf { -/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], /// ptr: *mut u8, /// } @@ -602,7 +598,7 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit<Self, Error> { /// try_pin_init!(Self { -/// big: Box::init(init::zeroed(), GFP_KERNEL)?, +/// big: KBox::init(init::zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -694,16 +690,16 @@ macro_rules! init { /// # Examples /// /// ```rust -/// use kernel::{init::{PinInit, zeroed}, error::Error}; +/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error}; /// struct BigBuf { -/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], /// } /// /// impl BigBuf { /// fn new() -> impl Init<Self, Error> { /// try_init!(Self { -/// big: Box::init(zeroed(), GFP_KERNEL)?, +/// big: KBox::init(zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// }? Error) /// } @@ -814,8 +810,8 @@ macro_rules! assert_pinned { /// A pin-initializer for the type `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the -/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. +/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use +/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. /// /// Also see the [module description](self). /// @@ -854,7 +850,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// # Examples /// /// ```rust - /// # #![allow(clippy::disallowed_names)] + /// # #![expect(clippy::disallowed_names)] /// use kernel::{types::Opaque, init::pin_init_from_closure}; /// #[repr(C)] /// struct RawFoo([u8; 16]); @@ -875,6 +871,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// } /// /// let foo = pin_init!(Foo { + /// // SAFETY: TODO. /// raw <- unsafe { /// Opaque::ffi_init(|s| { /// init_foo(s); @@ -894,7 +891,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { } /// An initializer returned by [`PinInit::pin_chain`]. -pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>); // SAFETY: The `__pinned_init` function is implemented such that it // - returns `Ok(())` on successful initialization, @@ -920,8 +917,8 @@ where /// An initializer for `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the -/// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because +/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use +/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. /// /// Also see the [module description](self). @@ -965,7 +962,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// # Examples /// /// ```rust - /// # #![allow(clippy::disallowed_names)] + /// # #![expect(clippy::disallowed_names)] /// use kernel::{types::Opaque, init::{self, init_from_closure}}; /// struct Foo { /// buf: [u8; 1_000_000], @@ -993,7 +990,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { } /// An initializer returned by [`Init::chain`]. -pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>); // SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, @@ -1077,8 +1074,9 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { /// # Examples /// /// ```rust -/// use kernel::{error::Error, init::init_array_from_fn}; -/// let array: Box<[usize; 1_000]> = Box::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); +/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn}; +/// let array: KBox<[usize; 1_000]> = +/// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` pub fn init_array_from_fn<I, const N: usize, T, E>( @@ -1162,6 +1160,7 @@ where // SAFETY: Every type can be initialized by-value. unsafe impl<T, E> Init<T, E> for T { unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: TODO. unsafe { slot.write(self) }; Ok(()) } @@ -1170,6 +1169,7 @@ unsafe impl<T, E> Init<T, E> for T { // SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. unsafe impl<T, E> PinInit<T, E> for T { unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: TODO. unsafe { self.__init(slot) } } } @@ -1243,26 +1243,6 @@ impl<T> InPlaceInit<T> for Arc<T> { } } -impl<T> InPlaceInit<T> for Box<T> { - type PinnedSelf = Pin<Self>; - - #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E> - where - E: From<AllocError>, - { - <Box<_> as BoxExt<_>>::new_uninit(flags)?.write_pin_init(init) - } - - #[inline] - fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> - where - E: From<AllocError>, - { - <Box<_> as BoxExt<_>>::new_uninit(flags)?.write_init(init) - } -} - impl<T> InPlaceInit<T> for UniqueArc<T> { type PinnedSelf = Pin<Self>; @@ -1299,28 +1279,6 @@ pub trait InPlaceWrite<T> { fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>; } -impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { - type Initialized = Box<T>; - - fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid. - unsafe { init.__init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }) - } - - fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { - let slot = self.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { self.assume_init() }.into()) - } -} - impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { type Initialized = UniqueArc<T>; @@ -1411,6 +1369,7 @@ pub fn zeroed<T: Zeroable>() -> impl Init<T> { macro_rules! impl_zeroable { ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. $(unsafe impl$($($generics)*)? Zeroable for $t {})* }; } @@ -1451,7 +1410,7 @@ impl_zeroable! { // // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. {<T: ?Sized>} Option<NonNull<T>>, - {<T: ?Sized>} Option<Box<T>>, + {<T: ?Sized>} Option<KBox<T>>, // SAFETY: `null` pointer is valid. // diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 13cefd37512f..74329cc3262c 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -15,9 +15,10 @@ use super::*; /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; -/// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this -/// type, since the closure needs to fulfill the same safety requirement as the -/// `__pinned_init`/`__init` functions. +/// Module-internal type implementing `PinInit` and `Init`. +/// +/// It is unsafe to create this type, since the closure needs to fulfill the same safety +/// requirement as the `__pinned_init`/`__init` functions. pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>); // SAFETY: While constructing the `InitClosure`, the user promised that it upholds the @@ -53,6 +54,7 @@ where pub unsafe trait HasPinData { type PinData: PinData; + #[expect(clippy::missing_safety_doc)] unsafe fn __pin_data() -> Self::PinData; } @@ -82,6 +84,7 @@ pub unsafe trait PinData: Copy { pub unsafe trait HasInitData { type InitData: InitData; + #[expect(clippy::missing_safety_doc)] unsafe fn __init_data() -> Self::InitData; } @@ -102,7 +105,7 @@ pub unsafe trait InitData: Copy { } } -pub struct AllData<T: ?Sized>(PhantomData<fn(Box<T>) -> Box<T>>); +pub struct AllData<T: ?Sized>(PhantomData<fn(KBox<T>) -> KBox<T>>); impl<T: ?Sized> Clone for AllData<T> { fn clone(&self) -> Self { @@ -112,10 +115,12 @@ impl<T: ?Sized> Clone for AllData<T> { impl<T: ?Sized> Copy for AllData<T> {} +// SAFETY: TODO. unsafe impl<T: ?Sized> InitData for AllData<T> { type Datee = T; } +// SAFETY: TODO. unsafe impl<T: ?Sized> HasInitData for T { type InitData = AllData<T>; diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index 9a0c4650ef67..1fd146a83241 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -182,13 +182,13 @@ //! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do //! // (normally people want to know if a type has any kind of drop glue at all, here we want //! // to know if it has any kind of custom drop glue, which is exactly what this bound does). -//! #[allow(drop_bounds)] +//! #[expect(drop_bounds)] //! impl<T: ::core::ops::Drop> MustNotImplDrop for T {} //! impl<T> MustNotImplDrop for Bar<T> {} //! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to //! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. -//! #[allow(non_camel_case_types)] +//! #[expect(non_camel_case_types)] //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} //! impl< //! T: ::kernel::init::PinnedDrop, @@ -513,6 +513,7 @@ macro_rules! __pinned_drop { } ), ) => { + // SAFETY: TODO. unsafe $($impl_sig)* { // Inherit all attributes and the type/ident tokens for the signature. $(#[$($attr)*])* @@ -872,6 +873,7 @@ macro_rules! __pin_data { } } + // SAFETY: TODO. unsafe impl<$($impl_generics)*> $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*> where $($whr)* @@ -923,14 +925,14 @@ macro_rules! __pin_data { // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, // if it also implements `Drop` trait MustNotImplDrop {} - #[allow(drop_bounds)] + #[expect(drop_bounds)] impl<T: ::core::ops::Drop> MustNotImplDrop for T {} impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> where $($whr)* {} // We also take care to prevent users from writing a useless `PinnedDrop` implementation. // They might implement `PinnedDrop` correctly for the struct, but forget to give // `PinnedDrop` as the parameter to `#[pin_data]`. - #[allow(non_camel_case_types)] + #[expect(non_camel_case_types)] trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} impl<T: $crate::init::PinnedDrop> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} @@ -987,6 +989,7 @@ macro_rules! __pin_data { // // The functions are `unsafe` to prevent accidentally calling them. #[allow(dead_code)] + #[expect(clippy::missing_safety_doc)] impl<$($impl_generics)*> $pin_data<$($ty_generics)*> where $($whr)* { @@ -997,6 +1000,7 @@ macro_rules! __pin_data { slot: *mut $p_type, init: impl $crate::init::PinInit<$p_type, E>, ) -> ::core::result::Result<(), E> { + // SAFETY: TODO. unsafe { $crate::init::PinInit::__pinned_init(init, slot) } } )* @@ -1007,6 +1011,7 @@ macro_rules! __pin_data { slot: *mut $type, init: impl $crate::init::Init<$type, E>, ) -> ::core::result::Result<(), E> { + // SAFETY: TODO. unsafe { $crate::init::Init::__init(init, slot) } } )* @@ -1121,6 +1126,8 @@ macro_rules! __init_internal { // no possibility of returning without `unsafe`. struct __InitOk; // Get the data about fields from the supplied type. + // + // SAFETY: TODO. let data = unsafe { use $crate::init::__internal::$has_data; // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal @@ -1176,6 +1183,7 @@ macro_rules! __init_internal { let init = move |slot| -> ::core::result::Result<(), $err> { init(slot).map(|__InitOk| ()) }; + // SAFETY: TODO. let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) }; init }}; @@ -1324,6 +1332,8 @@ macro_rules! __init_internal { // Endpoint, nothing more to munch, create the initializer. // Since we are in the closure that is never called, this will never get executed. // We abuse `slot` to get the correct type inference here: + // + // SAFETY: TODO. unsafe { // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal // information that is associated to already parsed fragments, so a path fragment diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs index cfa7d080b531..2fc7662339e5 100644 --- a/rust/kernel/ioctl.rs +++ b/rust/kernel/ioctl.rs @@ -4,7 +4,7 @@ //! //! C header: [`include/asm-generic/ioctl.h`](srctree/include/asm-generic/ioctl.h) -#![allow(non_snake_case)] +#![expect(non_snake_case)] use crate::build_assert; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 29947bb8593b..04dbee70d3e6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -12,10 +12,11 @@ //! do so first instead of bypassing this crate. #![no_std] +#![feature(arbitrary_self_types)] #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] -#![feature(new_uninit)] -#![feature(receiver_trait)] +#![feature(inline_const)] +#![feature(lint_reasons)] #![feature(unsize)] // Ensure conditional compilation based on the kernel configuration works; @@ -26,6 +27,8 @@ compile_error!("Missing kernel configuration for conditional compilation"); // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). extern crate self as kernel; +pub use ffi; + pub mod alloc; #[cfg(CONFIG_BLOCK)] pub mod block; @@ -45,6 +48,7 @@ pub mod list; #[cfg(CONFIG_NET)] pub mod net; pub mod page; +pub mod pid_namespace; pub mod prelude; pub mod print; pub mod rbtree; @@ -59,6 +63,7 @@ pub mod sync; pub mod task; pub mod time; pub mod tracepoint; +pub mod transmute; pub mod types; pub mod uaccess; pub mod workqueue; @@ -89,7 +94,7 @@ pub trait Module: Sized + Sync + Send { /// Equivalent to `THIS_MODULE` in the C API. /// -/// C header: [`include/linux/export.h`](srctree/include/linux/export.h) +/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) pub struct ThisModule(*mut bindings::module); // SAFETY: `THIS_MODULE` may be used from all threads within a module. diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 5b4aec29eb67..fb93330f4af4 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -354,6 +354,7 @@ impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> { /// /// `item` must not be in a different linked list (with the same id). pub unsafe fn remove(&mut self, item: &T) -> Option<ListArc<T, ID>> { + // SAFETY: TODO. let mut item = unsafe { ListLinks::fields(T::view_links(item)) }; // SAFETY: The user provided a reference, and reference are never dangling. // diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index d801b9dc6291..3483d8c232c4 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -441,9 +441,6 @@ where } } -// This is to allow [`ListArc`] (and variants) to be used as the type of `self`. -impl<T, const ID: u64> core::ops::Receiver for ListArc<T, ID> where T: ListArcSafe<ID> + ?Sized {} - // This is to allow coercion from `ListArc<T>` to `ListArc<U>` if `T` can be converted to the // dynamically-sized type (DST) `U`. impl<T, U, const ID: u64> core::ops::CoerceUnsized<ListArc<U, ID>> for ListArc<T, ID> diff --git a/rust/kernel/list/arc_field.rs b/rust/kernel/list/arc_field.rs index 2330f673427a..c4b9dd503982 100644 --- a/rust/kernel/list/arc_field.rs +++ b/rust/kernel/list/arc_field.rs @@ -56,7 +56,7 @@ impl<T, const ID: u64> ListArcField<T, ID> { /// /// The caller must have mutable access to the `ListArc<ID>` containing the struct with this /// field for the duration of the returned reference. - #[allow(clippy::mut_from_ref)] + #[expect(clippy::mut_from_ref)] pub unsafe fn assert_mut(&self) -> &mut T { // SAFETY: The caller has exclusive access to the `ListArc`, so they also have exclusive // access to this field. diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 801907fba199..b89c681d97c0 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -314,7 +314,7 @@ impl<T: Driver> Adapter<T> { /// `phydev` must be passed by the corresponding callback in `phy_driver`. unsafe extern "C" fn soft_reset_callback( phydev: *mut bindings::phy_device, - ) -> core::ffi::c_int { + ) -> crate::ffi::c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -328,7 +328,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn probe_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + unsafe extern "C" fn probe_callback(phydev: *mut bindings::phy_device) -> crate::ffi::c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we can exclusively access `phy_device` because @@ -345,7 +345,7 @@ impl<T: Driver> Adapter<T> { /// `phydev` must be passed by the corresponding callback in `phy_driver`. unsafe extern "C" fn get_features_callback( phydev: *mut bindings::phy_device, - ) -> core::ffi::c_int { + ) -> crate::ffi::c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -359,7 +359,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> crate::ffi::c_int { from_result(|| { // SAFETY: The C core code ensures that the accessors on // `Device` are okay to call even though `phy_device->lock` @@ -373,7 +373,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> crate::ffi::c_int { from_result(|| { // SAFETY: The C core code ensures that the accessors on // `Device` are okay to call even though `phy_device->lock` @@ -389,7 +389,7 @@ impl<T: Driver> Adapter<T> { /// `phydev` must be passed by the corresponding callback in `phy_driver`. unsafe extern "C" fn config_aneg_callback( phydev: *mut bindings::phy_device, - ) -> core::ffi::c_int { + ) -> crate::ffi::c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -405,7 +405,7 @@ impl<T: Driver> Adapter<T> { /// `phydev` must be passed by the corresponding callback in `phy_driver`. unsafe extern "C" fn read_status_callback( phydev: *mut bindings::phy_device, - ) -> core::ffi::c_int { + ) -> crate::ffi::c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -421,7 +421,7 @@ impl<T: Driver> Adapter<T> { /// `phydev` must be passed by the corresponding callback in `phy_driver`. unsafe extern "C" fn match_phy_device_callback( phydev: *mut bindings::phy_device, - ) -> core::ffi::c_int { + ) -> crate::ffi::c_int { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on // `Device` are okay to call. diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index 208a006d587c..fdac6c375fe4 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -20,6 +20,16 @@ pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; /// A bitmask that gives the page containing a given address. pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); +/// Round up the given number to the next multiple of [`PAGE_SIZE`]. +/// +/// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a +/// [`usize`]. +pub const fn page_align(addr: usize) -> usize { + // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong + // cases. + (addr + (PAGE_SIZE - 1)) & PAGE_MASK +} + /// A pointer to a page that owns the page allocation. /// /// # Invariants diff --git a/rust/kernel/pid_namespace.rs b/rust/kernel/pid_namespace.rs new file mode 100644 index 000000000000..0e93808e4639 --- /dev/null +++ b/rust/kernel/pid_namespace.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (c) 2024 Christian Brauner <brauner@kernel.org> + +//! Pid namespaces. +//! +//! C header: [`include/linux/pid_namespace.h`](srctree/include/linux/pid_namespace.h) and +//! [`include/linux/pid.h`](srctree/include/linux/pid.h) + +use crate::{ + bindings, + types::{AlwaysRefCounted, Opaque}, +}; +use core::ptr; + +/// Wraps the kernel's `struct pid_namespace`. Thread safe. +/// +/// This structure represents the Rust abstraction for a C `struct pid_namespace`. This +/// implementation abstracts the usage of an already existing C `struct pid_namespace` within Rust +/// code that we get passed from the C side. +#[repr(transparent)] +pub struct PidNamespace { + inner: Opaque<bindings::pid_namespace>, +} + +impl PidNamespace { + /// Returns a raw pointer to the inner C struct. + #[inline] + pub fn as_ptr(&self) -> *mut bindings::pid_namespace { + self.inner.get() + } + + /// Creates a reference to a [`PidNamespace`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` is valid and remains valid for the lifetime of the + /// returned [`PidNamespace`] reference. + pub unsafe fn from_ptr<'a>(ptr: *const bindings::pid_namespace) -> &'a Self { + // SAFETY: The safety requirements guarantee the validity of the dereference, while the + // `PidNamespace` type being transparent makes the cast ok. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: Instances of `PidNamespace` are always reference-counted. +unsafe impl AlwaysRefCounted for PidNamespace { + #[inline] + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_pid_ns(self.as_ptr()) }; + } + + #[inline] + unsafe fn dec_ref(obj: ptr::NonNull<PidNamespace>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_pid_ns(obj.cast().as_ptr()) } + } +} + +// SAFETY: +// - `PidNamespace::dec_ref` can be called from any thread. +// - It is okay to send ownership of `PidNamespace` across thread boundaries. +unsafe impl Send for PidNamespace {} + +// SAFETY: It's OK to access `PidNamespace` through shared references from other threads because +// we're either accessing properties that don't change or that are properly synchronised by C code. +unsafe impl Sync for PidNamespace {} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 4571daec0961..8bdab9aa0d16 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,10 +14,7 @@ #[doc(no_inline)] pub use core::pin::Pin; -pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt}; - -#[doc(no_inline)] -pub use alloc::{boxed::Box, vec::Vec}; +pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; #[doc(no_inline)] pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 508b0221256c..a28077a7cb30 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -14,6 +14,7 @@ use core::{ use crate::str::RawFormatter; // Called from `vsprintf` with format specifier `%pA`. +#[expect(clippy::missing_safety_doc)] #[no_mangle] unsafe extern "C" fn rust_fmt_argument( buf: *mut c_char, @@ -23,6 +24,7 @@ unsafe extern "C" fn rust_fmt_argument( use fmt::Write; // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`. let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) }; + // SAFETY: TODO. let _ = w.write_fmt(unsafe { *(ptr as *const fmt::Arguments<'_>) }); w.pos().cast() } @@ -102,6 +104,7 @@ pub unsafe fn call_printk( ) { // `_printk` does not seem to fail in any path. #[cfg(CONFIG_PRINTK)] + // SAFETY: TODO. unsafe { bindings::_printk( format_string.as_ptr() as _, @@ -137,7 +140,7 @@ pub fn call_printk_cont(args: fmt::Arguments<'_>) { #[doc(hidden)] #[cfg(not(testlib))] #[macro_export] -#[allow(clippy::crate_in_macro_def)] +#[expect(clippy::crate_in_macro_def)] macro_rules! print_macro ( // The non-continuation cases (most of them, e.g. `INFO`). ($format_string:path, false, $($arg:tt)+) => ( diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index 25eb36fd1cdc..cb4415a12258 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -7,7 +7,6 @@ //! Reference: <https://docs.kernel.org/core-api/rbtree.html> use crate::{alloc::Flags, bindings, container_of, error::Result, prelude::*}; -use alloc::boxed::Box; use core::{ cmp::{Ord, Ordering}, marker::PhantomData, @@ -497,7 +496,7 @@ impl<K, V> Drop for RBTree<K, V> { // but it is not observable. The loop invariant is still maintained. // SAFETY: `this` is valid per the loop invariant. - unsafe { drop(Box::from_raw(this.cast_mut())) }; + unsafe { drop(KBox::from_raw(this.cast_mut())) }; } } } @@ -764,7 +763,7 @@ impl<'a, K, V> Cursor<'a, K, V> { // point to the links field of `Node<K, V>` objects. let this = unsafe { container_of!(self.current.as_ptr(), Node<K, V>, links) }.cast_mut(); // SAFETY: `this` is valid by the type invariants as described above. - let node = unsafe { Box::from_raw(this) }; + let node = unsafe { KBox::from_raw(this) }; let node = RBTreeNode { node }; // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so // the tree cannot change. By the tree invariant, all nodes are valid. @@ -809,7 +808,7 @@ impl<'a, K, V> Cursor<'a, K, V> { // point to the links field of `Node<K, V>` objects. let this = unsafe { container_of!(neighbor, Node<K, V>, links) }.cast_mut(); // SAFETY: `this` is valid by the type invariants as described above. - let node = unsafe { Box::from_raw(this) }; + let node = unsafe { KBox::from_raw(this) }; return Some(RBTreeNode { node }); } None @@ -884,7 +883,8 @@ impl<'a, K, V> Cursor<'a, K, V> { NonNull::new(neighbor) } - /// SAFETY: + /// # Safety + /// /// - `node` must be a valid pointer to a node in an [`RBTree`]. /// - The caller has immutable access to `node` for the duration of 'b. unsafe fn to_key_value<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, &'b V) { @@ -894,7 +894,8 @@ impl<'a, K, V> Cursor<'a, K, V> { (k, unsafe { &*v }) } - /// SAFETY: + /// # Safety + /// /// - `node` must be a valid pointer to a node in an [`RBTree`]. /// - The caller has mutable access to `node` for the duration of 'b. unsafe fn to_key_value_mut<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, &'b mut V) { @@ -904,7 +905,8 @@ impl<'a, K, V> Cursor<'a, K, V> { (k, unsafe { &mut *v }) } - /// SAFETY: + /// # Safety + /// /// - `node` must be a valid pointer to a node in an [`RBTree`]. /// - The caller has immutable access to the key for the duration of 'b. unsafe fn to_key_value_raw<'b>(node: NonNull<bindings::rb_node>) -> (&'b K, *mut V) { @@ -1035,7 +1037,7 @@ impl<K, V> Iterator for IterRaw<K, V> { /// It contains the memory needed to hold a node that can be inserted into a red-black tree. One /// can be obtained by directly allocating it ([`RBTreeNodeReservation::new`]). pub struct RBTreeNodeReservation<K, V> { - node: Box<MaybeUninit<Node<K, V>>>, + node: KBox<MaybeUninit<Node<K, V>>>, } impl<K, V> RBTreeNodeReservation<K, V> { @@ -1043,7 +1045,7 @@ impl<K, V> RBTreeNodeReservation<K, V> { /// call to [`RBTree::insert`]. pub fn new(flags: Flags) -> Result<RBTreeNodeReservation<K, V>> { Ok(RBTreeNodeReservation { - node: <Box<_> as BoxExt<_>>::new_uninit(flags)?, + node: KBox::new_uninit(flags)?, }) } } @@ -1059,14 +1061,15 @@ impl<K, V> RBTreeNodeReservation<K, V> { /// Initialises a node reservation. /// /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. - pub fn into_node(mut self, key: K, value: V) -> RBTreeNode<K, V> { - self.node.write(Node { - key, - value, - links: bindings::rb_node::default(), - }); - // SAFETY: We just wrote to it. - let node = unsafe { self.node.assume_init() }; + pub fn into_node(self, key: K, value: V) -> RBTreeNode<K, V> { + let node = KBox::write( + self.node, + Node { + key, + value, + links: bindings::rb_node::default(), + }, + ); RBTreeNode { node } } } @@ -1076,7 +1079,7 @@ impl<K, V> RBTreeNodeReservation<K, V> { /// The node is fully initialised (with key and value) and can be inserted into a tree without any /// extra allocations or failure paths. pub struct RBTreeNode<K, V> { - node: Box<Node<K, V>>, + node: KBox<Node<K, V>>, } impl<K, V> RBTreeNode<K, V> { @@ -1088,7 +1091,9 @@ impl<K, V> RBTreeNode<K, V> { /// Get the key and value from inside the node. pub fn to_key_value(self) -> (K, V) { - (self.node.key, self.node.value) + let node = KBox::into_inner(self.node); + + (node.key, node.value) } } @@ -1110,7 +1115,7 @@ impl<K, V> RBTreeNode<K, V> { /// may be freed (but only for the key/value; memory for the node itself is kept for reuse). pub fn into_reservation(self) -> RBTreeNodeReservation<K, V> { RBTreeNodeReservation { - node: Box::drop_contents(self.node), + node: KBox::drop_contents(self.node), } } } @@ -1161,7 +1166,7 @@ impl<'a, K, V> RawVacantEntry<'a, K, V> { /// The `node` must have a key such that inserting it here does not break the ordering of this /// [`RBTree`]. fn insert(self, node: RBTreeNode<K, V>) -> &'a mut V { - let node = Box::into_raw(node.node); + let node = KBox::into_raw(node.node); // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when // the node is removed or replaced. @@ -1235,21 +1240,24 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { // SAFETY: The node was a node in the tree, but we removed it, so we can convert it // back into a box. node: unsafe { - Box::from_raw(container_of!(self.node_links, Node<K, V>, links).cast_mut()) + KBox::from_raw(container_of!(self.node_links, Node<K, V>, links).cast_mut()) }, } } /// Takes the value of the entry out of the map, and returns it. pub fn remove(self) -> V { - self.remove_node().node.value + let rb_node = self.remove_node(); + let node = KBox::into_inner(rb_node.node); + + node.value } /// Swap the current node for the provided node. /// /// The key of both nodes must be equal. fn replace(self, node: RBTreeNode<K, V>) -> RBTreeNode<K, V> { - let node = Box::into_raw(node.node); + let node = KBox::into_raw(node.node); // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when // the node is removed or replaced. @@ -1265,7 +1273,7 @@ impl<'a, K, V> OccupiedEntry<'a, K, V> { // - `self.node_ptr` produces a valid pointer to a node in the tree. // - Now that we removed this entry from the tree, we can convert the node to a box. let old_node = - unsafe { Box::from_raw(container_of!(self.node_links, Node<K, V>, links).cast_mut()) }; + unsafe { KBox::from_raw(container_of!(self.node_links, Node<K, V>, links).cast_mut()) }; RBTreeNode { node: old_node } } diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index 67bf9d37ddb5..279bd353687a 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +//! Rust standard library vendored code. +//! //! The contents of this file come from the Rust standard library, hosted in //! the <https://github.com/rust-lang/rust> repository, licensed under //! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, @@ -14,9 +16,9 @@ /// /// ```rust /// let a = 2; -/// # #[allow(clippy::dbg_macro)] +/// # #[expect(clippy::disallowed_macros)] /// let b = dbg!(a * 2) + 1; -/// // ^-- prints: [src/main.rs:2] a * 2 = 4 +/// // ^-- prints: [src/main.rs:3:9] a * 2 = 4 /// assert_eq!(b, 5); /// ``` /// @@ -52,7 +54,7 @@ /// With a method call: /// /// ```rust -/// # #[allow(clippy::dbg_macro)] +/// # #[expect(clippy::disallowed_macros)] /// fn foo(n: usize) { /// if dbg!(n.checked_sub(4)).is_some() { /// // ... @@ -65,14 +67,13 @@ /// This prints to the kernel log: /// /// ```text,ignore -/// [src/main.rs:4] n.checked_sub(4) = None +/// [src/main.rs:3:8] n.checked_sub(4) = None /// ``` /// /// Naive factorial implementation: /// /// ```rust -/// # #[allow(clippy::dbg_macro)] -/// # { +/// # #![expect(clippy::disallowed_macros)] /// fn factorial(n: u32) -> u32 { /// if dbg!(n <= 1) { /// dbg!(1) @@ -82,21 +83,20 @@ /// } /// /// dbg!(factorial(4)); -/// # } /// ``` /// /// This prints to the kernel log: /// /// ```text,ignore -/// [src/main.rs:3] n <= 1 = false -/// [src/main.rs:3] n <= 1 = false -/// [src/main.rs:3] n <= 1 = false -/// [src/main.rs:3] n <= 1 = true -/// [src/main.rs:4] 1 = 1 -/// [src/main.rs:5] n * factorial(n - 1) = 2 -/// [src/main.rs:5] n * factorial(n - 1) = 6 -/// [src/main.rs:5] n * factorial(n - 1) = 24 -/// [src/main.rs:11] factorial(4) = 24 +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = false +/// [src/main.rs:3:8] n <= 1 = true +/// [src/main.rs:4:9] 1 = 1 +/// [src/main.rs:5:9] n * factorial(n - 1) = 2 +/// [src/main.rs:5:9] n * factorial(n - 1) = 6 +/// [src/main.rs:5:9] n * factorial(n - 1) = 24 +/// [src/main.rs:11:1] factorial(4) = 24 /// ``` /// /// The `dbg!(..)` macro moves the input: @@ -118,7 +118,7 @@ /// a tuple (and return it, too): /// /// ``` -/// # #[allow(clippy::dbg_macro)] +/// # #![expect(clippy::disallowed_macros)] /// assert_eq!(dbg!(1usize, 2u32), (1, 2)); /// ``` /// @@ -127,11 +127,9 @@ /// invocations. You can use a 1-tuple directly if you need one: /// /// ``` -/// # #[allow(clippy::dbg_macro)] -/// # { +/// # #![expect(clippy::disallowed_macros)] /// assert_eq!(1, dbg!(1u32,)); // trailing comma ignored /// assert_eq!((1,), dbg!((1u32,))); // 1-tuple -/// # } /// ``` /// /// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index bb8d4f41475b..d04c12a1426d 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -2,8 +2,7 @@ //! String representations. -use crate::alloc::{flags::*, vec_ext::VecExt, AllocError}; -use alloc::vec::Vec; +use crate::alloc::{flags::*, AllocError, KVec}; use core::fmt::{self, Write}; use core::ops::{self, Deref, DerefMut, Index}; @@ -162,10 +161,10 @@ impl CStr { /// Returns the length of this string with `NUL`. #[inline] pub const fn len_with_nul(&self) -> usize { - // SAFETY: This is one of the invariant of `CStr`. - // We add a `unreachable_unchecked` here to hint the optimizer that - // the value returned from this function is non-zero. if self.0.is_empty() { + // SAFETY: This is one of the invariant of `CStr`. + // We add a `unreachable_unchecked` here to hint the optimizer that + // the value returned from this function is non-zero. unsafe { core::hint::unreachable_unchecked() }; } self.0.len() @@ -185,7 +184,7 @@ impl CStr { /// last at least `'a`. When `CStr` is alive, the memory pointed by `ptr` /// must not be mutated. #[inline] - pub unsafe fn from_char_ptr<'a>(ptr: *const core::ffi::c_char) -> &'a Self { + pub unsafe fn from_char_ptr<'a>(ptr: *const crate::ffi::c_char) -> &'a Self { // SAFETY: The safety precondition guarantees `ptr` is a valid pointer // to a `NUL`-terminated C string. let len = unsafe { bindings::strlen(ptr) } + 1; @@ -248,7 +247,7 @@ impl CStr { /// Returns a C pointer to the string. #[inline] - pub const fn as_char_ptr(&self) -> *const core::ffi::c_char { + pub const fn as_char_ptr(&self) -> *const crate::ffi::c_char { self.0.as_ptr() as _ } @@ -301,6 +300,7 @@ impl CStr { /// ``` #[inline] pub unsafe fn as_str_unchecked(&self) -> &str { + // SAFETY: TODO. unsafe { core::str::from_utf8_unchecked(self.as_bytes()) } } @@ -524,7 +524,28 @@ macro_rules! c_str { #[cfg(test)] mod tests { use super::*; - use alloc::format; + + struct String(CString); + + impl String { + fn from_fmt(args: fmt::Arguments<'_>) -> Self { + String(CString::try_from_fmt(args).unwrap()) + } + } + + impl Deref for String { + type Target = str; + + fn deref(&self) -> &str { + self.0.to_str().unwrap() + } + } + + macro_rules! format { + ($($f:tt)*) => ({ + &*String::from_fmt(kernel::fmt!($($f)*)) + }) + } const ALL_ASCII_CHARS: &'static str = "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ @@ -790,7 +811,7 @@ impl fmt::Write for Formatter { /// assert_eq!(s.is_ok(), false); /// ``` pub struct CString { - buf: Vec<u8>, + buf: KVec<u8>, } impl CString { @@ -803,7 +824,7 @@ impl CString { let size = f.bytes_written(); // Allocate a vector with the required number of bytes, and write to it. - let mut buf = <Vec<_> as VecExt<_>>::with_capacity(size, GFP_KERNEL)?; + let mut buf = KVec::with_capacity(size, GFP_KERNEL)?; // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; f.write_fmt(args)?; @@ -850,10 +871,9 @@ impl<'a> TryFrom<&'a CStr> for CString { type Error = AllocError; fn try_from(cstr: &'a CStr) -> Result<CString, AllocError> { - let mut buf = Vec::new(); + let mut buf = KVec::new(); - <Vec<_> as VecExt<_>>::extend_from_slice(&mut buf, cstr.as_bytes_with_nul(), GFP_KERNEL) - .map_err(|_| AllocError)?; + buf.extend_from_slice(cstr.as_bytes_with_nul(), GFP_KERNEL)?; // INVARIANT: The `CStr` and `CString` types have the same invariants for // the string data, and we copied it over without changes. diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index bae4a5179c72..1eab7ebf25fd 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -15,6 +15,7 @@ pub mod poll; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; +pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex}; pub use lock::spinlock::{new_spinlock, SpinLock}; pub use locked_by::LockedBy; diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 3021f30fd822..fa4509406ee9 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -17,13 +17,12 @@ //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ - alloc::{box_ext::BoxExt, AllocError, Flags}, + alloc::{AllocError, Flags, KBox}, bindings, init::{self, InPlaceInit, Init, PinInit}, try_init, types::{ForeignOwnable, Opaque}, }; -use alloc::boxed::Box; use core::{ alloc::Layout, fmt, @@ -171,9 +170,6 @@ impl<T: ?Sized> ArcInner<T> { } } -// This is to allow [`Arc`] (and variants) to be used as the type of `self`. -impl<T: ?Sized> core::ops::Receiver for Arc<T> {} - // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the // dynamically-sized type (DST) `U`. impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} @@ -204,11 +200,11 @@ impl<T> Arc<T> { data: contents, }; - let inner = <Box<_> as BoxExt<_>>::new(value, flags)?; + let inner = KBox::new(value, flags)?; // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. - Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) + Ok(unsafe { Self::from_inner(KBox::leak(inner).into()) }) } } @@ -336,12 +332,12 @@ impl<T: ?Sized> Arc<T> { impl<T: 'static> ForeignOwnable for Arc<T> { type Borrowed<'a> = ArcBorrow<'a, T>; - fn into_foreign(self) -> *const core::ffi::c_void { + fn into_foreign(self) -> *const crate::ffi::c_void { ManuallyDrop::new(self).ptr.as_ptr() as _ } - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> { - // SAFETY: By the safety requirement of this function, we know that `ptr` came from + unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> ArcBorrow<'a, T> { + // By the safety requirement of this function, we know that `ptr` came from // a previous call to `Arc::into_foreign`. let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap(); @@ -350,7 +346,7 @@ impl<T: 'static> ForeignOwnable for Arc<T> { unsafe { ArcBorrow::new(inner) } } - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { + unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self { // SAFETY: By the safety requirement of this function, we know that `ptr` came from // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and // holds a reference count increment that is transferrable to us. @@ -401,8 +397,8 @@ impl<T: ?Sized> Drop for Arc<T> { if is_zero { // The count reached zero, we must free the memory. // - // SAFETY: The pointer was initialised from the result of `Box::leak`. - unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; + // SAFETY: The pointer was initialised from the result of `KBox::leak`. + unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; } } } @@ -480,9 +476,6 @@ pub struct ArcBorrow<'a, T: ?Sized + 'a> { _p: PhantomData<&'a ()>, } -// This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`. -impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {} - // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into // `ArcBorrow<U>`. impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>> @@ -647,7 +640,7 @@ impl<T> UniqueArc<T> { /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. - let inner = Box::try_init::<AllocError>( + let inner = KBox::try_init::<AllocError>( try_init!(ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), @@ -657,8 +650,8 @@ impl<T> UniqueArc<T> { )?; Ok(UniqueArc { // INVARIANT: The newly-created object has a refcount of 1. - // SAFETY: The pointer from the `Box` is valid. - inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, + // SAFETY: The pointer from the `KBox` is valid. + inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) }, }) } } diff --git a/rust/kernel/sync/arc/std_vendor.rs b/rust/kernel/sync/arc/std_vendor.rs index a66a0c2831b3..11b3f4ecca5f 100644 --- a/rust/kernel/sync/arc/std_vendor.rs +++ b/rust/kernel/sync/arc/std_vendor.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +//! Rust standard library vendored code. +//! //! The contents of this file come from the Rust standard library, hosted in //! the <https://github.com/rust-lang/rust> repository, licensed under //! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index 2b306afbe56d..7df565038d7d 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -7,6 +7,7 @@ use super::{lock::Backend, lock::Guard, LockClassKey}; use crate::{ + ffi::{c_int, c_long}, init::PinInit, pin_init, str::CStr, @@ -14,7 +15,6 @@ use crate::{ time::Jiffies, types::Opaque, }; -use core::ffi::{c_int, c_long}; use core::marker::PhantomPinned; use core::ptr; use macros::pin_data; @@ -70,8 +70,8 @@ pub use new_condvar; /// } /// /// /// Allocates a new boxed `Example`. -/// fn new_example() -> Result<Pin<Box<Example>>> { -/// Box::pin_init(pin_init!(Example { +/// fn new_example() -> Result<Pin<KBox<Example>>> { +/// KBox::pin_init(pin_init!(Example { /// value <- new_mutex!(0), /// value_changed <- new_condvar!(), /// }), GFP_KERNEL) @@ -93,7 +93,6 @@ pub struct CondVar { } // SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread. -#[allow(clippy::non_send_fields_in_send_ty)] unsafe impl Send for CondVar {} // SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index d6e9bab114b8..41dcddac69e2 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -18,6 +18,9 @@ use macros::pin_data; pub mod mutex; pub mod spinlock; +pub(super) mod global; +pub use global::{GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; + /// The "backend" of a lock. /// /// It is the actual implementation of the lock, without the need to repeat patterns used in all @@ -51,7 +54,7 @@ pub unsafe trait Backend { /// remain valid for read indefinitely. unsafe fn init( ptr: *mut Self::State, - name: *const core::ffi::c_char, + name: *const crate::ffi::c_char, key: *mut bindings::lock_class_key, ); @@ -63,6 +66,13 @@ pub unsafe trait Backend { #[must_use] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState; + /// Tries to acquire the lock. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState>; + /// Releases the lock, giving up its ownership. /// /// # Safety @@ -133,6 +143,15 @@ impl<T: ?Sized, B: Backend> Lock<T, B> { // SAFETY: The lock was just acquired. unsafe { Guard::new(self, state) } } + + /// Tries to acquire the lock. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + pub fn try_lock(&self) -> Option<Guard<'_, T, B>> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + unsafe { B::try_lock(self.state.get()).map(|state| Guard::new(self, state)) } + } } /// A lock guard. @@ -155,9 +174,9 @@ impl<T: ?Sized, B: Backend> Guard<'_, T, B> { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; - // SAFETY: The lock was just unlocked above and is being relocked now. - let _relock = - ScopeGuard::new(|| unsafe { B::relock(self.lock.state.get(), &mut self.state) }); + let _relock = ScopeGuard::new(|| + // SAFETY: The lock was just unlocked above and is being relocked now. + unsafe { B::relock(self.lock.state.get(), &mut self.state) }); cb() } diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs new file mode 100644 index 000000000000..480ee724e3cc --- /dev/null +++ b/rust/kernel/sync/lock/global.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Support for defining statics containing locks. + +use crate::{ + str::CStr, + sync::lock::{Backend, Guard, Lock}, + sync::{LockClassKey, LockedBy}, + types::Opaque, +}; +use core::{ + cell::UnsafeCell, + marker::{PhantomData, PhantomPinned}, +}; + +/// Trait implemented for marker types for global locks. +/// +/// See [`global_lock!`] for examples. +pub trait GlobalLockBackend { + /// The name for this global lock. + const NAME: &'static CStr; + /// Item type stored in this global lock. + type Item: 'static; + /// The backend used for this global lock. + type Backend: Backend + 'static; + /// The class for this global lock. + fn get_lock_class() -> &'static LockClassKey; +} + +/// Type used for global locks. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalLock<B: GlobalLockBackend> { + inner: Lock<B::Item, B::Backend>, +} + +impl<B: GlobalLockBackend> GlobalLock<B> { + /// Creates a global lock. + /// + /// # Safety + /// + /// * Before any other method on this lock is called, [`Self::init`] must be called. + /// * The type `B` must not be used with any other lock. + pub const unsafe fn new(data: B::Item) -> Self { + Self { + inner: Lock { + state: Opaque::uninit(), + data: UnsafeCell::new(data), + _pin: PhantomPinned, + }, + } + } + + /// Initializes a global lock. + /// + /// # Safety + /// + /// Must not be called more than once on a given lock. + pub unsafe fn init(&'static self) { + // SAFETY: The pointer to `state` is valid for the duration of this call, and both `name` + // and `key` are valid indefinitely. The `state` is pinned since we have a `'static` + // reference to `self`. + // + // We have exclusive access to the `state` since the caller of `new` promised to call + // `init` before using any other methods. As `init` can only be called once, all other + // uses of this lock must happen after this call. + unsafe { + B::Backend::init( + self.inner.state.get(), + B::NAME.as_char_ptr(), + B::get_lock_class().as_ptr(), + ) + } + } + + /// Lock this global lock. + pub fn lock(&'static self) -> GlobalGuard<B> { + GlobalGuard { + inner: self.inner.lock(), + } + } + + /// Try to lock this global lock. + pub fn try_lock(&'static self) -> Option<GlobalGuard<B>> { + Some(GlobalGuard { + inner: self.inner.try_lock()?, + }) + } +} + +/// A guard for a [`GlobalLock`]. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalGuard<B: GlobalLockBackend> { + inner: Guard<'static, B::Item, B::Backend>, +} + +impl<B: GlobalLockBackend> core::ops::Deref for GlobalGuard<B> { + type Target = B::Item; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl<B: GlobalLockBackend> core::ops::DerefMut for GlobalGuard<B> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +/// A version of [`LockedBy`] for a [`GlobalLock`]. +/// +/// See [`global_lock!`] for examples. +pub struct GlobalLockedBy<T: ?Sized, B: GlobalLockBackend> { + _backend: PhantomData<B>, + value: UnsafeCell<T>, +} + +// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`. +unsafe impl<T, B> Send for GlobalLockedBy<T, B> +where + T: ?Sized, + B: GlobalLockBackend, + LockedBy<T, B::Item>: Send, +{ +} + +// SAFETY: The same thread-safety rules as `LockedBy` apply to `GlobalLockedBy`. +unsafe impl<T, B> Sync for GlobalLockedBy<T, B> +where + T: ?Sized, + B: GlobalLockBackend, + LockedBy<T, B::Item>: Sync, +{ +} + +impl<T, B: GlobalLockBackend> GlobalLockedBy<T, B> { + /// Create a new [`GlobalLockedBy`]. + /// + /// The provided value will be protected by the global lock indicated by `B`. + pub fn new(val: T) -> Self { + Self { + value: UnsafeCell::new(val), + _backend: PhantomData, + } + } +} + +impl<T: ?Sized, B: GlobalLockBackend> GlobalLockedBy<T, B> { + /// Access the value immutably. + /// + /// The caller must prove shared access to the lock. + pub fn as_ref<'a>(&'a self, _guard: &'a GlobalGuard<B>) -> &'a T { + // SAFETY: The lock is globally unique, so there can only be one guard. + unsafe { &*self.value.get() } + } + + /// Access the value mutably. + /// + /// The caller must prove shared exclusive to the lock. + pub fn as_mut<'a>(&'a self, _guard: &'a mut GlobalGuard<B>) -> &'a mut T { + // SAFETY: The lock is globally unique, so there can only be one guard. + unsafe { &mut *self.value.get() } + } + + /// Access the value mutably directly. + /// + /// The caller has exclusive access to this `GlobalLockedBy`, so they do not need to hold the + /// lock. + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } +} + +/// Defines a global lock. +/// +/// The global mutex must be initialized before first use. Usually this is done by calling +/// [`GlobalLock::init`] in the module initializer. +/// +/// # Examples +/// +/// A global counter: +/// +/// ``` +/// # mod ex { +/// # use kernel::prelude::*; +/// kernel::sync::global_lock! { +/// // SAFETY: Initialized in module initializer before first use. +/// unsafe(uninit) static MY_COUNTER: Mutex<u32> = 0; +/// } +/// +/// fn increment_counter() -> u32 { +/// let mut guard = MY_COUNTER.lock(); +/// *guard += 1; +/// *guard +/// } +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result<Self> { +/// // SAFETY: Called exactly once. +/// unsafe { MY_COUNTER.init() }; +/// +/// Ok(MyModule {}) +/// } +/// } +/// # struct MyModule {} +/// # } +/// ``` +/// +/// A global mutex used to protect all instances of a given struct: +/// +/// ``` +/// # mod ex { +/// # use kernel::prelude::*; +/// use kernel::sync::{GlobalGuard, GlobalLockedBy}; +/// +/// kernel::sync::global_lock! { +/// // SAFETY: Initialized in module initializer before first use. +/// unsafe(uninit) static MY_MUTEX: Mutex<()> = (); +/// } +/// +/// /// All instances of this struct are protected by `MY_MUTEX`. +/// struct MyStruct { +/// my_counter: GlobalLockedBy<u32, MY_MUTEX>, +/// } +/// +/// impl MyStruct { +/// /// Increment the counter in this instance. +/// /// +/// /// The caller must hold the `MY_MUTEX` mutex. +/// fn increment(&self, guard: &mut GlobalGuard<MY_MUTEX>) -> u32 { +/// let my_counter = self.my_counter.as_mut(guard); +/// *my_counter += 1; +/// *my_counter +/// } +/// } +/// +/// impl kernel::Module for MyModule { +/// fn init(_module: &'static ThisModule) -> Result<Self> { +/// // SAFETY: Called exactly once. +/// unsafe { MY_MUTEX.init() }; +/// +/// Ok(MyModule {}) +/// } +/// } +/// # struct MyModule {} +/// # } +/// ``` +#[macro_export] +macro_rules! global_lock { + { + $(#[$meta:meta])* $pub:vis + unsafe(uninit) static $name:ident: $kind:ident<$valuety:ty> = $value:expr; + } => { + #[doc = ::core::concat!( + "Backend type used by [`", + ::core::stringify!($name), + "`](static@", + ::core::stringify!($name), + ")." + )] + #[allow(non_camel_case_types, unreachable_pub)] + $pub enum $name {} + + impl $crate::sync::lock::GlobalLockBackend for $name { + const NAME: &'static $crate::str::CStr = $crate::c_str!(::core::stringify!($name)); + type Item = $valuety; + type Backend = $crate::global_lock_inner!(backend $kind); + + fn get_lock_class() -> &'static $crate::sync::LockClassKey { + $crate::static_lock_class!() + } + } + + $(#[$meta])* + $pub static $name: $crate::sync::lock::GlobalLock<$name> = { + // Defined here to be outside the unsafe scope. + let init: $valuety = $value; + + // SAFETY: + // * The user of this macro promises to initialize the macro before use. + // * We are only generating one static with this backend type. + unsafe { $crate::sync::lock::GlobalLock::new(init) } + }; + }; +} +pub use global_lock; + +#[doc(hidden)] +#[macro_export] +macro_rules! global_lock_inner { + (backend Mutex) => { + $crate::sync::lock::mutex::MutexBackend + }; + (backend SpinLock) => { + $crate::sync::lock::spinlock::SpinLockBackend + }; +} diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 30632070ee67..0e946ebefce1 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -58,7 +58,7 @@ pub use new_mutex; /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); @@ -96,7 +96,7 @@ unsafe impl super::Backend for MutexBackend { unsafe fn init( ptr: *mut Self::State, - name: *const core::ffi::c_char, + name: *const crate::ffi::c_char, key: *mut bindings::lock_class_key, ) { // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and @@ -115,4 +115,15 @@ unsafe impl super::Backend for MutexBackend { // caller is the owner of the mutex. unsafe { bindings::mutex_unlock(ptr) }; } + + unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::mutex_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index ea5c5bc1ce12..9f4d128bed98 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -56,7 +56,7 @@ pub use new_spinlock; /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); @@ -95,7 +95,7 @@ unsafe impl super::Backend for SpinLockBackend { unsafe fn init( ptr: *mut Self::State, - name: *const core::ffi::c_char, + name: *const crate::ffi::c_char, key: *mut bindings::lock_class_key, ) { // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and @@ -114,4 +114,15 @@ unsafe impl super::Backend for SpinLockBackend { // caller is the owner of the spinlock. unsafe { bindings::spin_unlock(ptr) } } + + unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } } diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index ce2ee8d87865..a7b244675c2b 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -43,7 +43,7 @@ use core::{cell::UnsafeCell, mem::size_of, ptr}; /// struct InnerDirectory { /// /// The sum of the bytes used by all files. /// bytes_used: u64, -/// _files: Vec<File>, +/// _files: KVec<File>, /// } /// /// struct Directory { diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 080599075875..07bc22a7645c 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -6,11 +6,12 @@ use crate::{ bindings, - types::{NotThreadSafe, Opaque}, + ffi::{c_int, c_long, c_uint}, + pid_namespace::PidNamespace, + types::{ARef, NotThreadSafe, Opaque}, }; use core::{ cmp::{Eq, PartialEq}, - ffi::{c_int, c_long, c_uint}, ops::Deref, ptr, }; @@ -36,6 +37,16 @@ macro_rules! current { }; } +/// Returns the currently running task's pid namespace. +#[macro_export] +macro_rules! current_pid_ns { + () => { + // SAFETY: Deref + addr-of below create a temporary `PidNamespaceRef` that cannot outlive + // the caller. + unsafe { &*$crate::task::Task::current_pid_ns() } + }; +} + /// Wraps the kernel's `struct task_struct`. /// /// # Invariants @@ -145,6 +156,97 @@ impl Task { } } + /// Returns a PidNamespace reference for the currently executing task's/thread's pid namespace. + /// + /// This function can be used to create an unbounded lifetime by e.g., storing the returned + /// PidNamespace in a global variable which would be a bug. So the recommended way to get the + /// current task's/thread's pid namespace is to use the [`current_pid_ns`] macro because it is + /// safe. + /// + /// # Safety + /// + /// Callers must ensure that the returned object doesn't outlive the current task/thread. + pub unsafe fn current_pid_ns() -> impl Deref<Target = PidNamespace> { + struct PidNamespaceRef<'a> { + task: &'a PidNamespace, + _not_send: NotThreadSafe, + } + + impl Deref for PidNamespaceRef<'_> { + type Target = PidNamespace; + + fn deref(&self) -> &Self::Target { + self.task + } + } + + // The lifetime of `PidNamespace` is bound to `Task` and `struct pid`. + // + // The `PidNamespace` of a `Task` doesn't ever change once the `Task` is alive. A + // `unshare(CLONE_NEWPID)` or `setns(fd_pidns/pidfd, CLONE_NEWPID)` will not have an effect + // on the calling `Task`'s pid namespace. It will only effect the pid namespace of children + // created by the calling `Task`. This invariant guarantees that after having acquired a + // reference to a `Task`'s pid namespace it will remain unchanged. + // + // When a task has exited and been reaped `release_task()` will be called. This will set + // the `PidNamespace` of the task to `NULL`. So retrieving the `PidNamespace` of a task + // that is dead will return `NULL`. Note, that neither holding the RCU lock nor holding a + // referencing count to + // the `Task` will prevent `release_task()` being called. + // + // In order to retrieve the `PidNamespace` of a `Task` the `task_active_pid_ns()` function + // can be used. There are two cases to consider: + // + // (1) retrieving the `PidNamespace` of the `current` task + // (2) retrieving the `PidNamespace` of a non-`current` task + // + // From system call context retrieving the `PidNamespace` for case (1) is always safe and + // requires neither RCU locking nor a reference count to be held. Retrieving the + // `PidNamespace` after `release_task()` for current will return `NULL` but no codepath + // like that is exposed to Rust. + // + // Retrieving the `PidNamespace` from system call context for (2) requires RCU protection. + // Accessing `PidNamespace` outside of RCU protection requires a reference count that + // must've been acquired while holding the RCU lock. Note that accessing a non-`current` + // task means `NULL` can be returned as the non-`current` task could have already passed + // through `release_task()`. + // + // To retrieve (1) the `current_pid_ns!()` macro should be used which ensure that the + // returned `PidNamespace` cannot outlive the calling scope. The associated + // `current_pid_ns()` function should not be called directly as it could be abused to + // created an unbounded lifetime for `PidNamespace`. The `current_pid_ns!()` macro allows + // Rust to handle the common case of accessing `current`'s `PidNamespace` without RCU + // protection and without having to acquire a reference count. + // + // For (2) the `task_get_pid_ns()` method must be used. This will always acquire a + // reference on `PidNamespace` and will return an `Option` to force the caller to + // explicitly handle the case where `PidNamespace` is `None`, something that tends to be + // forgotten when doing the equivalent operation in `C`. Missing RCU primitives make it + // difficult to perform operations that are otherwise safe without holding a reference + // count as long as RCU protection is guaranteed. But it is not important currently. But we + // do want it in the future. + // + // Note for (2) the required RCU protection around calling `task_active_pid_ns()` + // synchronizes against putting the last reference of the associated `struct pid` of + // `task->thread_pid`. The `struct pid` stored in that field is used to retrieve the + // `PidNamespace` of the caller. When `release_task()` is called `task->thread_pid` will be + // `NULL`ed and `put_pid()` on said `struct pid` will be delayed in `free_pid()` via + // `call_rcu()` allowing everyone with an RCU protected access to the `struct pid` acquired + // from `task->thread_pid` to finish. + // + // SAFETY: The current task's pid namespace is valid as long as the current task is running. + let pidns = unsafe { bindings::task_active_pid_ns(Task::current_raw()) }; + PidNamespaceRef { + // SAFETY: If the current thread is still running, the current task and its associated + // pid namespace are valid. `PidNamespaceRef` is not `Send`, so we know it cannot be + // transferred to another thread (where it could potentially outlive the current + // `Task`). The caller needs to ensure that the PidNamespaceRef doesn't outlive the + // current task/thread. + task: unsafe { PidNamespace::from_ptr(pidns) }, + _not_send: NotThreadSafe, + } + } + /// Returns a raw pointer to the task. #[inline] pub fn as_ptr(&self) -> *mut bindings::task_struct { @@ -188,11 +290,32 @@ impl Task { unsafe { bindings::signal_pending(self.as_ptr()) != 0 } } - /// Returns the given task's pid in the current pid namespace. - pub fn pid_in_current_ns(&self) -> Pid { - // SAFETY: It's valid to pass a null pointer as the namespace (defaults to current - // namespace). The task pointer is also valid. - unsafe { bindings::task_tgid_nr_ns(self.as_ptr(), ptr::null_mut()) } + /// Returns task's pid namespace with elevated reference count + pub fn get_pid_ns(&self) -> Option<ARef<PidNamespace>> { + // SAFETY: By the type invariant, we know that `self.0` is valid. + let ptr = unsafe { bindings::task_get_pid_ns(self.as_ptr()) }; + if ptr.is_null() { + None + } else { + // SAFETY: `ptr` is valid by the safety requirements of this function. And we own a + // reference count via `task_get_pid_ns()`. + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::pid_namespace`. + Some(unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(ptr.cast::<PidNamespace>())) }) + } + } + + /// Returns the given task's pid in the provided pid namespace. + #[doc(alias = "task_tgid_nr_ns")] + pub fn tgid_nr_ns(&self, pidns: Option<&PidNamespace>) -> Pid { + let pidns = match pidns { + Some(pidns) => pidns.as_ptr(), + None => core::ptr::null_mut(), + }; + // SAFETY: By the type invariant, we know that `self.0` is valid. We received a valid + // PidNamespace that we can use as a pointer or we received an empty PidNamespace and + // thus pass a null pointer. The underlying C function is safe to be used with NULL + // pointers. + unsafe { bindings::task_tgid_nr_ns(self.as_ptr(), pidns) } } /// Wakes up the task. diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index e3bb5e89f88d..379c0f5772e5 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -12,10 +12,10 @@ pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; /// The time unit of Linux kernel. One jiffy equals (1/HZ) second. -pub type Jiffies = core::ffi::c_ulong; +pub type Jiffies = crate::ffi::c_ulong; /// The millisecond time unit. -pub type Msecs = core::ffi::c_uint; +pub type Msecs = crate::ffi::c_uint; /// Converts milliseconds to jiffies. #[inline] diff --git a/rust/kernel/transmute.rs b/rust/kernel/transmute.rs new file mode 100644 index 000000000000..1c7d43771a37 --- /dev/null +++ b/rust/kernel/transmute.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Traits for transmuting types. + +/// Types for which any bit pattern is valid. +/// +/// Not all types are valid for all values. For example, a `bool` must be either zero or one, so +/// reading arbitrary bytes into something that contains a `bool` is not okay. +/// +/// It's okay for the type to have padding, as initializing those bytes has no effect. +/// +/// # Safety +/// +/// All bit-patterns must be valid for this type. This type must not have interior mutability. +pub unsafe trait FromBytes {} + +macro_rules! impl_frombytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? FromBytes for $t {})* + }; +} + +impl_frombytes! { + // SAFETY: All bit patterns are acceptable values of the types below. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + + // SAFETY: If all bit patterns are acceptable for individual values in an array, then all bit + // patterns are also acceptable for arrays of that type. + {<T: FromBytes>} [T], + {<T: FromBytes, const N: usize>} [T; N], +} + +/// Types that can be viewed as an immutable slice of initialized bytes. +/// +/// If a struct implements this trait, then it is okay to copy it byte-for-byte to userspace. This +/// means that it should not have any padding, as padding bytes are uninitialized. Reading +/// uninitialized memory is not just undefined behavior, it may even lead to leaking sensitive +/// information on the stack to userspace. +/// +/// The struct should also not hold kernel pointers, as kernel pointer addresses are also considered +/// sensitive. However, leaking kernel pointers is not considered undefined behavior by Rust, so +/// this is a correctness requirement, but not a safety requirement. +/// +/// # Safety +/// +/// Values of this type may not contain any uninitialized bytes. This type must not have interior +/// mutability. +pub unsafe trait AsBytes {} + +macro_rules! impl_asbytes { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? AsBytes for $t {})* + }; +} + +impl_asbytes! { + // SAFETY: Instances of the following types have no uninitialized portions. + u8, u16, u32, u64, usize, + i8, i16, i32, i64, isize, + bool, + char, + str, + + // SAFETY: If individual values in an array have no uninitialized portions, then the array + // itself does not have any uninitialized portions either. + {<T: AsBytes>} [T], + {<T: AsBytes, const N: usize>} [T; N], +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 3238ffaab031..7a133d938551 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -3,13 +3,11 @@ //! Kernel types. use crate::init::{self, PinInit}; -use alloc::boxed::Box; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, - pin::Pin, ptr::NonNull, }; @@ -31,7 +29,7 @@ pub trait ForeignOwnable: Sized { /// For example, it might be invalid, dangling or pointing to uninitialized memory. Using it in /// any way except for [`ForeignOwnable::from_foreign`], [`ForeignOwnable::borrow`], /// [`ForeignOwnable::try_from_foreign`] can result in undefined behavior. - fn into_foreign(self) -> *const core::ffi::c_void; + fn into_foreign(self) -> *const crate::ffi::c_void; /// Borrows a foreign-owned object. /// @@ -39,7 +37,7 @@ pub trait ForeignOwnable: Sized { /// /// `ptr` must have been returned by a previous call to [`ForeignOwnable::into_foreign`] for /// which a previous matching [`ForeignOwnable::from_foreign`] hasn't been called yet. - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Self::Borrowed<'a>; + unsafe fn borrow<'a>(ptr: *const crate::ffi::c_void) -> Self::Borrowed<'a>; /// Converts a foreign-owned object back to a Rust-owned one. /// @@ -49,7 +47,7 @@ pub trait ForeignOwnable: Sized { /// which a previous matching [`ForeignOwnable::from_foreign`] hasn't been called yet. /// Additionally, all instances (if any) of values returned by [`ForeignOwnable::borrow`] for /// this object must have been dropped. - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self; + unsafe fn from_foreign(ptr: *const crate::ffi::c_void) -> Self; /// Tries to convert a foreign-owned object back to a Rust-owned one. /// @@ -60,7 +58,7 @@ pub trait ForeignOwnable: Sized { /// /// `ptr` must either be null or satisfy the safety requirements for /// [`ForeignOwnable::from_foreign`]. - unsafe fn try_from_foreign(ptr: *const core::ffi::c_void) -> Option<Self> { + unsafe fn try_from_foreign(ptr: *const crate::ffi::c_void) -> Option<Self> { if ptr.is_null() { None } else { @@ -71,64 +69,16 @@ pub trait ForeignOwnable: Sized { } } -impl<T: 'static> ForeignOwnable for Box<T> { - type Borrowed<'a> = &'a T; - - fn into_foreign(self) -> *const core::ffi::c_void { - Box::into_raw(self) as _ - } - - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements of `from_foreign` also ensure that the object remains alive for - // the lifetime of the returned value. - unsafe { &*ptr.cast() } - } - - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous - // call to `Self::into_foreign`. - unsafe { Box::from_raw(ptr as _) } - } -} - -impl<T: 'static> ForeignOwnable for Pin<Box<T>> { - type Borrowed<'a> = Pin<&'a T>; - - fn into_foreign(self) -> *const core::ffi::c_void { - // SAFETY: We are still treating the box as pinned. - Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ - } - - unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Pin<&'a T> { - // SAFETY: The safety requirements for this function ensure that the object is still alive, - // so it is safe to dereference the raw pointer. - // The safety requirements of `from_foreign` also ensure that the object remains alive for - // the lifetime of the returned value. - let r = unsafe { &*ptr.cast() }; - - // SAFETY: This pointer originates from a `Pin<Box<T>>`. - unsafe { Pin::new_unchecked(r) } - } - - unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { - // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous - // call to `Self::into_foreign`. - unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } - } -} - impl ForeignOwnable for () { type Borrowed<'a> = (); - fn into_foreign(self) -> *const core::ffi::c_void { + fn into_foreign(self) -> *const crate::ffi::c_void { core::ptr::NonNull::dangling().as_ptr() } - unsafe fn borrow<'a>(_: *const core::ffi::c_void) -> Self::Borrowed<'a> {} + unsafe fn borrow<'a>(_: *const crate::ffi::c_void) -> Self::Borrowed<'a> {} - unsafe fn from_foreign(_: *const core::ffi::c_void) -> Self {} + unsafe fn from_foreign(_: *const crate::ffi::c_void) -> Self {} } /// Runs a cleanup function/closure when dropped. @@ -185,7 +135,7 @@ impl ForeignOwnable for () { /// # use kernel::types::ScopeGuard; /// fn example3(arg: bool) -> Result { /// let mut vec = -/// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len())); +/// ScopeGuard::new_with_data(KVec::new(), |v| pr_info!("vec had {} elements\n", v.len())); /// /// vec.push(10u8, GFP_KERNEL)?; /// if arg { @@ -225,7 +175,7 @@ impl<T, F: FnOnce(T)> ScopeGuard<T, F> { impl ScopeGuard<(), fn(())> { /// Creates a new guarded object with the given cleanup function. pub fn new(cleanup: impl FnOnce()) -> ScopeGuard<(), impl FnOnce(())> { - ScopeGuard::new_with_data((), move |_| cleanup()) + ScopeGuard::new_with_data((), move |()| cleanup()) } } @@ -256,7 +206,58 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> { /// Stores an opaque value. /// -/// This is meant to be used with FFI objects that are never interpreted by Rust code. +/// `Opaque<T>` is meant to be used with FFI objects that are never interpreted by Rust code. +/// +/// It is used to wrap structs from the C side, like for example `Opaque<bindings::mutex>`. +/// It gets rid of all the usual assumptions that Rust has for a value: +/// +/// * The value is allowed to be uninitialized (for example have invalid bit patterns: `3` for a +/// [`bool`]). +/// * The value is allowed to be mutated, when a `&Opaque<T>` exists on the Rust side. +/// * No uniqueness for mutable references: it is fine to have multiple `&mut Opaque<T>` point to +/// the same value. +/// * The value is not allowed to be shared with other threads (i.e. it is `!Sync`). +/// +/// This has to be used for all values that the C side has access to, because it can't be ensured +/// that the C side is adhering to the usual constraints that Rust needs. +/// +/// Using `Opaque<T>` allows to continue to use references on the Rust side even for values shared +/// with C. +/// +/// # Examples +/// +/// ``` +/// # #![expect(unreachable_pub, clippy::disallowed_names)] +/// use kernel::types::Opaque; +/// # // Emulate a C struct binding which is from C, maybe uninitialized or not, only the C side +/// # // knows. +/// # mod bindings { +/// # pub struct Foo { +/// # pub val: u8, +/// # } +/// # } +/// +/// // `foo.val` is assumed to be handled on the C side, so we use `Opaque` to wrap it. +/// pub struct Foo { +/// foo: Opaque<bindings::Foo>, +/// } +/// +/// impl Foo { +/// pub fn get_val(&self) -> u8 { +/// let ptr = Opaque::get(&self.foo); +/// +/// // SAFETY: `Self` is valid from C side. +/// unsafe { (*ptr).val } +/// } +/// } +/// +/// // Create an instance of `Foo` with the `Opaque` wrapper. +/// let foo = Foo { +/// foo: Opaque::new(bindings::Foo { val: 0xdb }), +/// }; +/// +/// assert_eq!(foo.get_val(), 0xdb); +/// ``` #[repr(transparent)] pub struct Opaque<T> { value: UnsafeCell<MaybeUninit<T>>, @@ -410,6 +411,7 @@ impl<T: AlwaysRefCounted> ARef<T> { /// /// struct Empty {} /// + /// # // SAFETY: TODO. /// unsafe impl AlwaysRefCounted for Empty { /// fn inc_ref(&self) {} /// unsafe fn dec_ref(_obj: NonNull<Self>) {} @@ -417,6 +419,7 @@ impl<T: AlwaysRefCounted> ARef<T> { /// /// let mut data = Empty {}; /// let ptr = NonNull::<Empty>::new(&mut data as *mut _).unwrap(); + /// # // SAFETY: TODO. /// let data_ref: ARef<Empty> = unsafe { ARef::from_raw(ptr) }; /// let raw_ptr: NonNull<Empty> = ARef::into_raw(data_ref); /// @@ -461,6 +464,15 @@ impl<T: AlwaysRefCounted> Drop for ARef<T> { } /// A sum type that always holds either a value of type `L` or `R`. +/// +/// # Examples +/// +/// ``` +/// use kernel::types::Either; +/// +/// let left_value: Either<i32, &str> = Either::Left(7); +/// let right_value: Either<i32, &str> = Either::Right("right value"); +/// ``` pub enum Either<L, R> { /// Constructs an instance of [`Either`] containing a value of type `L`. Left(L), @@ -469,70 +481,6 @@ pub enum Either<L, R> { Right(R), } -/// Types for which any bit pattern is valid. -/// -/// Not all types are valid for all values. For example, a `bool` must be either zero or one, so -/// reading arbitrary bytes into something that contains a `bool` is not okay. -/// -/// It's okay for the type to have padding, as initializing those bytes has no effect. -/// -/// # Safety -/// -/// All bit-patterns must be valid for this type. This type must not have interior mutability. -pub unsafe trait FromBytes {} - -// SAFETY: All bit patterns are acceptable values of the types below. -unsafe impl FromBytes for u8 {} -unsafe impl FromBytes for u16 {} -unsafe impl FromBytes for u32 {} -unsafe impl FromBytes for u64 {} -unsafe impl FromBytes for usize {} -unsafe impl FromBytes for i8 {} -unsafe impl FromBytes for i16 {} -unsafe impl FromBytes for i32 {} -unsafe impl FromBytes for i64 {} -unsafe impl FromBytes for isize {} -// SAFETY: If all bit patterns are acceptable for individual values in an array, then all bit -// patterns are also acceptable for arrays of that type. -unsafe impl<T: FromBytes> FromBytes for [T] {} -unsafe impl<T: FromBytes, const N: usize> FromBytes for [T; N] {} - -/// Types that can be viewed as an immutable slice of initialized bytes. -/// -/// If a struct implements this trait, then it is okay to copy it byte-for-byte to userspace. This -/// means that it should not have any padding, as padding bytes are uninitialized. Reading -/// uninitialized memory is not just undefined behavior, it may even lead to leaking sensitive -/// information on the stack to userspace. -/// -/// The struct should also not hold kernel pointers, as kernel pointer addresses are also considered -/// sensitive. However, leaking kernel pointers is not considered undefined behavior by Rust, so -/// this is a correctness requirement, but not a safety requirement. -/// -/// # Safety -/// -/// Values of this type may not contain any uninitialized bytes. This type must not have interior -/// mutability. -pub unsafe trait AsBytes {} - -// SAFETY: Instances of the following types have no uninitialized portions. -unsafe impl AsBytes for u8 {} -unsafe impl AsBytes for u16 {} -unsafe impl AsBytes for u32 {} -unsafe impl AsBytes for u64 {} -unsafe impl AsBytes for usize {} -unsafe impl AsBytes for i8 {} -unsafe impl AsBytes for i16 {} -unsafe impl AsBytes for i32 {} -unsafe impl AsBytes for i64 {} -unsafe impl AsBytes for isize {} -unsafe impl AsBytes for bool {} -unsafe impl AsBytes for char {} -unsafe impl AsBytes for str {} -// SAFETY: If individual values in an array have no uninitialized portions, then the array itself -// does not have any uninitialized portions either. -unsafe impl<T: AsBytes> AsBytes for [T] {} -unsafe impl<T: AsBytes, const N: usize> AsBytes for [T; N] {} - /// Zero-sized type to mark types not [`Send`]. /// /// Add this type as a field to your struct if your type should not be sent to a different task. diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index e9347cff99ab..05b0b8d13b10 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -8,11 +8,10 @@ use crate::{ alloc::Flags, bindings, error::Result, + ffi::{c_ulong, c_void}, prelude::*, - types::{AsBytes, FromBytes}, + transmute::{AsBytes, FromBytes}, }; -use alloc::vec::Vec; -use core::ffi::{c_ulong, c_void}; use core::mem::{size_of, MaybeUninit}; /// The type used for userspace addresses. @@ -46,15 +45,14 @@ pub type UserPtr = usize; /// every byte in the region. /// /// ```no_run -/// use alloc::vec::Vec; -/// use core::ffi::c_void; +/// use kernel::ffi::c_void; /// use kernel::error::Result; /// use kernel::uaccess::{UserPtr, UserSlice}; /// /// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result<()> { /// let (read, mut write) = UserSlice::new(uptr, len).reader_writer(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// for b in &mut buf { @@ -69,8 +67,7 @@ pub type UserPtr = usize; /// Example illustrating a TOCTOU (time-of-check to time-of-use) bug. /// /// ```no_run -/// use alloc::vec::Vec; -/// use core::ffi::c_void; +/// use kernel::ffi::c_void; /// use kernel::error::{code::EINVAL, Result}; /// use kernel::uaccess::{UserPtr, UserSlice}; /// @@ -78,21 +75,21 @@ pub type UserPtr = usize; /// fn is_valid(uptr: UserPtr, len: usize) -> Result<bool> { /// let read = UserSlice::new(uptr, len).reader(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// todo!() /// } /// /// /// Returns the bytes behind this user pointer if they are valid. -/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result<Vec<u8>> { +/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result<KVec<u8>> { /// if !is_valid(uptr, len)? { /// return Err(EINVAL); /// } /// /// let read = UserSlice::new(uptr, len).reader(); /// -/// let mut buf = Vec::new(); +/// let mut buf = KVec::new(); /// read.read_all(&mut buf, GFP_KERNEL)?; /// /// // THIS IS A BUG! The bytes could have changed since we checked them. @@ -130,7 +127,7 @@ impl UserSlice { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(self, buf: &mut Vec<u8>, flags: Flags) -> Result { + pub fn read_all(self, buf: &mut KVec<u8>, flags: Flags) -> Result { self.reader().read_all(buf, flags) } @@ -291,9 +288,9 @@ impl UserSliceReader { /// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. - pub fn read_all(mut self, buf: &mut Vec<u8>, flags: Flags) -> Result { + pub fn read_all(mut self, buf: &mut KVec<u8>, flags: Flags) -> Result { let len = self.length; - VecExt::<u8>::reserve(buf, len, flags)?; + buf.reserve(len, flags)?; // The call to `try_reserve` was successful, so the spare capacity is at least `len` bytes // long. diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 553a5cba2adc..4d1d2062f6eb 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -216,7 +216,7 @@ impl Queue { func: Some(func), }); - self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?); + self.enqueue(KBox::pin_init(init, flags).map_err(|_| AllocError)?); Ok(()) } } @@ -239,9 +239,9 @@ impl<T> ClosureWork<T> { } impl<T: FnOnce()> WorkItem for ClosureWork<T> { - type Pointer = Pin<Box<Self>>; + type Pointer = Pin<KBox<Self>>; - fn run(mut this: Pin<Box<Self>>) { + fn run(mut this: Pin<KBox<Self>>) { if let Some(func) = this.as_mut().project().take() { (func)() } @@ -297,7 +297,7 @@ pub unsafe trait RawWorkItem<const ID: u64> { /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin<Box<T>>` and [`Arc<T>`], and is mainly intended to be +/// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be /// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] /// instead. The [`run`] method on this trait will usually just perform the appropriate /// `container_of` translation and then call into the [`run`][WorkItem::run] method from the @@ -329,7 +329,7 @@ pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> { /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. pub trait WorkItem<const ID: u64 = 0> { /// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or - /// `Pin<Box<Self>>`. + /// `Pin<KBox<Self>>`. type Pointer: WorkItemPointer<ID>; /// The method that should be called when this work item is executed. @@ -366,7 +366,6 @@ unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {} impl<T: ?Sized, const ID: u64> Work<T, ID> { /// Creates a new instance of [`Work`]. #[inline] - #[allow(clippy::new_ret_no_self)] pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> where T: WorkItem<ID>, @@ -520,13 +519,14 @@ impl_has_work! { impl{T} HasWork<Self> for ClosureWork<T> { self.work } } +// SAFETY: TODO. unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { - // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr as *mut Work<T, ID>; // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; @@ -537,6 +537,7 @@ where } } +// SAFETY: TODO. unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T> where T: WorkItem<ID, Pointer = Self>, @@ -565,18 +566,19 @@ where } } -unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>> +// SAFETY: TODO. +unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { - // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr as *mut Work<T, ID>; // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. - let boxed = unsafe { Box::from_raw(ptr) }; + let boxed = unsafe { KBox::from_raw(ptr) }; // SAFETY: The box was already pinned when it was enqueued. let pinned = unsafe { Pin::new_unchecked(boxed) }; @@ -584,7 +586,8 @@ where } } -unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>> +// SAFETY: TODO. +unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<KBox<T>> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, @@ -598,9 +601,9 @@ where // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily // remove the `Pin` wrapper. let boxed = unsafe { Pin::into_inner_unchecked(self) }; - let ptr = Box::into_raw(boxed); + let ptr = KBox::into_raw(boxed); - // SAFETY: Pointers into a `Box` point at a valid value. + // SAFETY: Pointers into a `KBox` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index a626b1145e5c..4ab94e44adfe 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -30,7 +30,7 @@ use proc_macro::TokenStream; /// /// # Examples /// -/// ```ignore +/// ``` /// use kernel::prelude::*; /// /// module!{ @@ -42,22 +42,16 @@ use proc_macro::TokenStream; /// alias: ["alternate_module_name"], /// } /// -/// struct MyModule; +/// struct MyModule(i32); /// /// impl kernel::Module for MyModule { -/// fn init() -> Result<Self> { -/// // If the parameter is writeable, then the kparam lock must be -/// // taken to read the parameter: -/// { -/// let lock = THIS_MODULE.kernel_param_lock(); -/// pr_info!("i32 param is: {}\n", writeable_i32.read(&lock)); -/// } -/// // If the parameter is read only, it can be read without locking -/// // the kernel parameters: -/// pr_info!("i32 param is: {}\n", my_i32.read()); -/// Ok(Self) +/// fn init(_module: &'static ThisModule) -> Result<Self> { +/// let foo: i32 = 42; +/// pr_info!("I contain: {}\n", foo); +/// Ok(Self(foo)) /// } /// } +/// # fn main() {} /// ``` /// /// ## Firmware @@ -69,7 +63,7 @@ use proc_macro::TokenStream; /// build an initramfs uses this information to put the firmware files into /// the initramfs image. /// -/// ```ignore +/// ``` /// use kernel::prelude::*; /// /// module!{ @@ -84,10 +78,11 @@ use proc_macro::TokenStream; /// struct MyDeviceDriverModule; /// /// impl kernel::Module for MyDeviceDriverModule { -/// fn init() -> Result<Self> { +/// fn init(_module: &'static ThisModule) -> Result<Self> { /// Ok(Self) /// } /// } +/// # fn main() {} /// ``` /// /// # Supported argument types @@ -132,7 +127,7 @@ pub fn module(ts: TokenStream) -> TokenStream { /// calls to this function at compile time: /// /// ```compile_fail -/// # use kernel::error::VTABLE_DEFAULT_ERROR; +/// # // Intentionally missing `use`s to simplify `rusttest`. /// kernel::build_error(VTABLE_DEFAULT_ERROR) /// ``` /// @@ -142,7 +137,7 @@ pub fn module(ts: TokenStream) -> TokenStream { /// /// # Examples /// -/// ```ignore +/// ``` /// use kernel::error::VTABLE_DEFAULT_ERROR; /// use kernel::prelude::*; /// @@ -187,12 +182,27 @@ pub fn vtable(attr: TokenStream, ts: TokenStream) -> TokenStream { /// /// # Examples /// -/// ```ignore -/// use kernel::macro::concat_idents; +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; +/// use kernel::macros::concat_idents; /// /// macro_rules! pub_no_prefix { /// ($prefix:ident, $($newname:ident),+) => { -/// $(pub(crate) const $newname: u32 = kernel::macros::concat_idents!($prefix, $newname);)+ +/// $(pub(crate) const $newname: u32 = concat_idents!($prefix, $newname);)+ /// }; /// } /// @@ -238,21 +248,35 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// /// # Examples /// -/// ```rust,ignore +/// ``` +/// # #![feature(lint_reasons)] +/// # use kernel::prelude::*; +/// # use std::{sync::Mutex, process::Command}; +/// # use kernel::macros::pin_data; /// #[pin_data] /// struct DriverData { /// #[pin] -/// queue: Mutex<Vec<Command>>, -/// buf: Box<[u8; 1024 * 1024]>, +/// queue: Mutex<KVec<Command>>, +/// buf: KBox<[u8; 1024 * 1024]>, /// } /// ``` /// -/// ```rust,ignore +/// ``` +/// # #![feature(lint_reasons)] +/// # use kernel::prelude::*; +/// # use std::{sync::Mutex, process::Command}; +/// # use core::pin::Pin; +/// # pub struct Info; +/// # mod bindings { +/// # pub unsafe fn destroy_info(_ptr: *mut super::Info) {} +/// # } +/// use kernel::macros::{pin_data, pinned_drop}; +/// /// #[pin_data(PinnedDrop)] /// struct DriverData { /// #[pin] -/// queue: Mutex<Vec<Command>>, -/// buf: Box<[u8; 1024 * 1024]>, +/// queue: Mutex<KVec<Command>>, +/// buf: KBox<[u8; 1024 * 1024]>, /// raw_info: *mut Info, /// } /// @@ -262,6 +286,7 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// unsafe { bindings::destroy_info(self.raw_info) }; /// } /// } +/// # fn main() {} /// ``` /// /// [`pin_init!`]: ../kernel/macro.pin_init.html @@ -277,13 +302,22 @@ pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { /// /// # Examples /// -/// ```rust,ignore +/// ``` +/// # #![feature(lint_reasons)] +/// # use kernel::prelude::*; +/// # use macros::{pin_data, pinned_drop}; +/// # use std::{sync::Mutex, process::Command}; +/// # use core::pin::Pin; +/// # mod bindings { +/// # pub struct Info; +/// # pub unsafe fn destroy_info(_ptr: *mut Info) {} +/// # } /// #[pin_data(PinnedDrop)] /// struct DriverData { /// #[pin] -/// queue: Mutex<Vec<Command>>, -/// buf: Box<[u8; 1024 * 1024]>, -/// raw_info: *mut Info, +/// queue: Mutex<KVec<Command>>, +/// buf: KBox<[u8; 1024 * 1024]>, +/// raw_info: *mut bindings::Info, /// } /// /// #[pinned_drop] @@ -309,12 +343,25 @@ pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { /// /// # Example /// -/// ```ignore -/// use kernel::macro::paste; -/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; /// macro_rules! pub_no_prefix { /// ($prefix:ident, $($newname:ident),+) => { -/// paste! { +/// kernel::macros::paste! { /// $(pub(crate) const $newname: u32 = [<$prefix $newname>];)+ /// } /// }; @@ -353,13 +400,26 @@ pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { /// * `lower`: change the identifier to lower case. /// * `upper`: change the identifier to upper case. /// -/// ```ignore -/// use kernel::macro::paste; -/// +/// ``` +/// # const binder_driver_return_protocol_BR_OK: u32 = 0; +/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1; +/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2; +/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3; +/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4; +/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5; +/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6; +/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7; +/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8; +/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9; +/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10; +/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11; +/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12; +/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13; +/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14; /// macro_rules! pub_no_prefix { /// ($prefix:ident, $($newname:ident),+) => { /// kernel::macros::paste! { -/// $(pub(crate) const fn [<$newname:lower:span>]: u32 = [<$prefix $newname:span>];)+ +/// $(pub(crate) const fn [<$newname:lower:span>]() -> u32 { [<$prefix $newname:span>] })+ /// } /// }; /// } @@ -390,7 +450,7 @@ pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { /// /// Literals can also be concatenated with other identifiers: /// -/// ```ignore +/// ``` /// macro_rules! create_numbered_fn { /// ($name:literal, $val:literal) => { /// kernel::macros::paste! { @@ -418,7 +478,9 @@ pub fn paste(input: TokenStream) -> TokenStream { /// /// # Examples /// -/// ```rust,ignore +/// ``` +/// use kernel::macros::Zeroable; +/// /// #[derive(Zeroable)] /// pub struct DriverData { /// id: i64, diff --git a/rust/macros/module.rs b/rust/macros/module.rs index aef3b132f32b..e7a087b7e884 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -253,7 +253,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[doc(hidden)] #[no_mangle] #[link_section = \".init.text\"] - pub unsafe extern \"C\" fn init_module() -> core::ffi::c_int {{ + pub unsafe extern \"C\" fn init_module() -> kernel::ffi::c_int {{ // SAFETY: This function is inaccessible to the outside due to the double // module wrapping it. It is called exactly once by the C side via its // unique name. @@ -292,7 +292,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[doc(hidden)] #[link_section = \"{initcall_section}\"] #[used] - pub static __{name}_initcall: extern \"C\" fn() -> core::ffi::c_int = __{name}_init; + pub static __{name}_initcall: extern \"C\" fn() -> kernel::ffi::c_int = __{name}_init; #[cfg(not(MODULE))] #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] @@ -307,7 +307,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(not(MODULE))] #[doc(hidden)] #[no_mangle] - pub extern \"C\" fn __{name}_init() -> core::ffi::c_int {{ + pub extern \"C\" fn __{name}_init() -> kernel::ffi::c_int {{ // SAFETY: This function is inaccessible to the outside due to the double // module wrapping it. It is called exactly once by the C side via its // placement above in the initcall section. @@ -330,7 +330,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { /// # Safety /// /// This function must only be called once. - unsafe fn __init() -> core::ffi::c_int {{ + unsafe fn __init() -> kernel::ffi::c_int {{ match <{type_} as kernel::Module>::init(&super::super::THIS_MODULE) {{ Ok(m) => {{ // SAFETY: No data race, since `__MOD` can only be accessed by this diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs index f40d42b35b58..6529a387673f 100644 --- a/rust/macros/paste.rs +++ b/rust/macros/paste.rs @@ -2,7 +2,7 @@ use proc_macro::{Delimiter, Group, Ident, Spacing, Span, TokenTree}; -fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { +fn concat_helper(tokens: &[TokenTree]) -> Vec<(String, Span)> { let mut tokens = tokens.iter(); let mut segments = Vec::new(); let mut span = None; @@ -46,12 +46,21 @@ fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { }; segments.push((value, sp)); } - _ => panic!("unexpected token in paste segments"), + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => { + let tokens = group.stream().into_iter().collect::<Vec<TokenTree>>(); + segments.append(&mut concat_helper(tokens.as_slice())); + } + token => panic!("unexpected token in paste segments: {:?}", token), }; } + segments +} + +fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree { + let segments = concat_helper(tokens); let pasted: String = segments.into_iter().map(|x| x.0).collect(); - TokenTree::Ident(Ident::new(&pasted, span.unwrap_or(group_span))) + TokenTree::Ident(Ident::new(&pasted, group_span)) } pub(crate) fn expand(tokens: &mut Vec<TokenTree>) { diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index 80a00260e3e7..13495910271f 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -14,6 +14,7 @@ #![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] #![allow( clippy::all, + clippy::undocumented_unsafe_blocks, dead_code, missing_docs, non_camel_case_types, @@ -24,4 +25,9 @@ unsafe_op_in_unsafe_fn )] +// Manual definition of blocklisted types. +type __kernel_size_t = usize; +type __kernel_ssize_t = isize; +type __kernel_ptrdiff_t = isize; + include!(concat!(env!("OBJTREE"), "/rust/uapi/uapi_generated.rs")); diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs index 2a9eaab62d1c..4aaf117bf8e3 100644 --- a/samples/rust/rust_minimal.rs +++ b/samples/rust/rust_minimal.rs @@ -13,7 +13,7 @@ module! { } struct RustMinimal { - numbers: Vec<i32>, + numbers: KVec<i32>, } impl kernel::Module for RustMinimal { @@ -21,7 +21,7 @@ impl kernel::Module for RustMinimal { pr_info!("Rust minimal sample (init)\n"); pr_info!("Am I built-in? {}\n", !cfg!(MODULE)); - let mut numbers = Vec::new(); + let mut numbers = KVec::new(); numbers.push(72, GFP_KERNEL)?; numbers.push(108, GFP_KERNEL)?; numbers.push(200, GFP_KERNEL)?; diff --git a/samples/rust/rust_print_main.rs b/samples/rust/rust_print_main.rs index 6d14b08cac1c..aed90a6feecf 100644 --- a/samples/rust/rust_print_main.rs +++ b/samples/rust/rust_print_main.rs @@ -15,6 +15,7 @@ module! { struct RustPrint; +#[expect(clippy::disallowed_macros)] fn arc_print() -> Result { use kernel::sync::*; diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 03ee558fcd4d..f483a54380e7 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -248,7 +248,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := asm_const,asm_goto,new_uninit +rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree @@ -259,7 +259,7 @@ rust_common_cmd = \ -Zallow-features=$(rust_allowed_features) \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ - -Zunstable-options --extern force:alloc --extern kernel \ + -Zunstable-options --extern kernel \ --crate-type rlib -L $(objtree)/rust/ \ --crate-name $(basename $(notdir $@)) \ --sysroot=/dev/null \ diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index d2bc63cde8c6..09e1d166d8d2 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -65,13 +65,6 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs): ) append_crate( - "alloc", - sysroot_src / "alloc" / "src" / "lib.rs", - ["core", "compiler_builtins"], - cfg=crates_cfgs.get("alloc", []), - ) - - append_crate( "macros", srctree / "rust" / "macros" / "lib.rs", [], @@ -96,7 +89,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs): append_crate( "kernel", srctree / "rust" / "kernel" / "lib.rs", - ["core", "alloc", "macros", "build_error", "bindings"], + ["core", "macros", "build_error", "bindings"], cfg=cfg, ) crates[-1]["source"] = { @@ -133,7 +126,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs): append_crate( name, path, - ["core", "alloc", "kernel"], + ["core", "kernel"], cfg=cfg, ) diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index 5262c56dd674..93c0ef7fb3fb 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -225,6 +225,21 @@ if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then exit 1 fi +if [ "$bindgen_libclang_cversion" -ge 1900100 ] && + [ "$rust_bindings_generator_cversion" -lt 6905 ]; then + # Distributions may have patched the issue (e.g. Debian did). + if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang_concat.h | grep -q foofoo; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' < 0.69.5 together with libclang >= 19.1" + echo >&2 "*** may not work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/2824)," + echo >&2 "*** unless patched (like Debian's)." + echo >&2 "*** Your bindgen version: $rust_bindings_generator_version" + echo >&2 "*** Your libclang version: $bindgen_libclang_version" + echo >&2 "***" + warning=1 + fi +fi + # If the C compiler is Clang, then we can also check whether its version # matches the `libclang` version used by the Rust bindings generator. # diff --git a/scripts/rust_is_available_bindgen_libclang_concat.h b/scripts/rust_is_available_bindgen_libclang_concat.h new file mode 100644 index 000000000000..efc6e98d0f1d --- /dev/null +++ b/scripts/rust_is_available_bindgen_libclang_concat.h @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#define F(x) int x##x +F(foo); diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index 413741037fb3..4fcc319dea84 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -54,7 +54,7 @@ else: """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched=False): + def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched=False, libclang_concat_patched=False): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: @@ -65,12 +65,19 @@ else: else: version_0_66_case = "raise SystemExit(1)" + if libclang_concat_patched: + libclang_concat_case = "print('pub static mut foofoo: ::std::os::raw::c_int;')" + else: + libclang_concat_case = "pass" + return cls.generate_executable(f"""#!/usr/bin/env python3 import sys if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): {libclang_case} elif "rust_is_available_bindgen_0_66.h" in " ".join(sys.argv): {version_0_66_case} +elif "rust_is_available_bindgen_libclang_concat.h" in " ".join(sys.argv): + {libclang_concat_case} else: print({repr(version_stdout)}) """) @@ -268,6 +275,31 @@ else: result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) + def test_bindgen_bad_libclang_concat(self): + for (bindgen_version, libclang_version, expected_not_patched) in ( + ("0.69.4", "18.0.0", self.Expected.SUCCESS), + ("0.69.4", "19.1.0", self.Expected.SUCCESS_WITH_WARNINGS), + ("0.69.4", "19.2.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.69.5", "18.0.0", self.Expected.SUCCESS), + ("0.69.5", "19.1.0", self.Expected.SUCCESS), + ("0.69.5", "19.2.0", self.Expected.SUCCESS), + + ("0.70.0", "18.0.0", self.Expected.SUCCESS), + ("0.70.0", "19.1.0", self.Expected.SUCCESS), + ("0.70.0", "19.2.0", self.Expected.SUCCESS), + ): + with self.subTest(bindgen_version=bindgen_version, libclang_version=libclang_version): + cc = self.generate_clang(f"clang version {libclang_version}") + libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {libclang_version} [-W#pragma-messages], err: false" + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr) + result = self.run_script(expected_not_patched, { "BINDGEN": bindgen, "CC": cc }) + if expected_not_patched == self.Expected.SUCCESS_WITH_WARNINGS: + self.assertIn(f"Rust bindings generator '{bindgen}' < 0.69.5 together with libclang >= 19.1", result.stderr) + + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_concat_patched=True) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen, "CC": cc }) + def test_clang_matches_bindgen_libclang_different_bindgen(self): bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) diff --git a/sound/pci/korg1212/korg1212.c b/sound/pci/korg1212/korg1212.c index e62fb1ad6d77..49b71082c485 100644 --- a/sound/pci/korg1212/korg1212.c +++ b/sound/pci/korg1212/korg1212.c @@ -2108,7 +2108,7 @@ static int snd_korg1212_create(struct snd_card *card, struct pci_dev *pci) for (i=0; i<kAudioChannels; i++) korg1212->volumePhase[i] = 0; - err = pcim_iomap_regions_request_all(pci, 1 << 0, "korg1212"); + err = pcim_request_all_regions(pci, "korg1212"); if (err < 0) return err; @@ -2130,7 +2130,9 @@ static int snd_korg1212_create(struct snd_card *card, struct pci_dev *pci) korg1212->iomem2, iomem2_size, stateName[korg1212->cardState]); - korg1212->iobase = pcim_iomap_table(pci)[0]; + korg1212->iobase = pcim_iomap(pci, 0, 0); + if (!korg1212->iobase) + return -ENOMEM; err = devm_request_irq(&pci->dev, pci->irq, snd_korg1212_interrupt, IRQF_SHARED, diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index 2ebfb826dcea..bca47d136f05 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -30,9 +30,7 @@ endef # FEATURE_TESTS_BASIC := \ backtrace \ - dwarf \ - dwarf_getlocations \ - dwarf_getcfi \ + libdw \ eventfd \ fortify-source \ get_current_dir_name \ @@ -61,7 +59,6 @@ FEATURE_TESTS_BASIC := \ reallocarray \ stackprotector-all \ timerfd \ - libdw-dwarf-unwind \ zlib \ lzma \ get_cpuid \ @@ -121,8 +118,7 @@ ifeq ($(FEATURE_TESTS),all) endif FEATURE_DISPLAY ?= \ - dwarf \ - dwarf_getlocations \ + libdw \ glibc \ libbfd \ libbfd-buildid \ @@ -134,7 +130,6 @@ FEATURE_DISPLAY ?= \ libpython \ libcrypto \ libunwind \ - libdw-dwarf-unwind \ libcapstone \ llvm-perf \ zlib \ @@ -234,7 +229,7 @@ endef # # generates feature value assignment for name, like: -# $(call feature_assign,dwarf) == feature-dwarf=1 +# $(call feature_assign,libdw) == feature-libdw=1 # feature_assign = feature-$(1)=$(feature-$(1)) diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 28cdfb11a755..043dfd00fce7 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -5,9 +5,7 @@ FILES= \ test-all.bin \ test-backtrace.bin \ test-bionic.bin \ - test-dwarf.bin \ - test-dwarf_getlocations.bin \ - test-dwarf_getcfi.bin \ + test-libdw.bin \ test-eventfd.bin \ test-fortify-source.bin \ test-get_current_dir_name.bin \ @@ -53,7 +51,6 @@ FILES= \ test-pthread-barrier.bin \ test-stackprotector-all.bin \ test-timerfd.bin \ - test-libdw-dwarf-unwind.bin \ test-libbabeltrace.bin \ test-libcapstone.bin \ test-compile-32.bin \ @@ -169,9 +166,9 @@ $(OUTPUT)test-libopencsd.bin: $(BUILD) # -lopencsd_c_api -lopencsd provided by # $(FEATURE_CHECK_LDFLAGS-libopencsd) -DWARFLIBS := -ldw +DWLIBS := -ldw ifeq ($(findstring -static,${LDFLAGS}),-static) - DWARFLIBS += -lelf -lz -llzma -lbz2 -lzstd + DWLIBS += -lelf -lz -llzma -lbz2 -lzstd LIBDW_VERSION := $(shell $(PKG_CONFIG) --modversion libdw).0.0 LIBDW_VERSION_1 := $(word 1, $(subst ., ,$(LIBDW_VERSION))) @@ -180,21 +177,15 @@ ifeq ($(findstring -static,${LDFLAGS}),-static) # Elfutils merged libebl.a into libdw.a starting from version 0.177, # Link libebl.a only if libdw is older than this version. ifeq ($(shell test $(LIBDW_VERSION_2) -lt 177; echo $$?),0) - DWARFLIBS += -lebl + DWLIBS += -lebl endif # Must put -ldl after -lebl for dependency DWARFLIBS += -ldl endif -$(OUTPUT)test-dwarf.bin: - $(BUILD) $(DWARFLIBS) - -$(OUTPUT)test-dwarf_getlocations.bin: - $(BUILD) $(DWARFLIBS) - -$(OUTPUT)test-dwarf_getcfi.bin: - $(BUILD) $(DWARFLIBS) +$(OUTPUT)test-libdw.bin: + $(BUILD) $(DWLIBS) $(OUTPUT)test-libelf-getphdrnum.bin: $(BUILD) -lelf @@ -321,9 +312,6 @@ $(OUTPUT)test-backtrace.bin: $(OUTPUT)test-timerfd.bin: $(BUILD) -$(OUTPUT)test-libdw-dwarf-unwind.bin: - $(BUILD) # -ldw provided by $(FEATURE_CHECK_LDFLAGS-libdw-dwarf-unwind) - $(OUTPUT)test-libbabeltrace.bin: $(BUILD) # -lbabeltrace provided by $(FEATURE_CHECK_LDFLAGS-libbabeltrace) diff --git a/tools/build/feature/test-all.c b/tools/build/feature/test-all.c index 6f4bf386a3b5..59ef3d7fe6a4 100644 --- a/tools/build/feature/test-all.c +++ b/tools/build/feature/test-all.c @@ -38,12 +38,8 @@ # include "test-glibc.c" #undef main -#define main main_test_dwarf -# include "test-dwarf.c" -#undef main - -#define main main_test_dwarf_getlocations -# include "test-dwarf_getlocations.c" +#define main main_test_libdw +# include "test-libdw.c" #undef main #define main main_test_eventfd @@ -98,10 +94,6 @@ # include "test-stackprotector-all.c" #undef main -#define main main_test_libdw_dwarf_unwind -# include "test-libdw-dwarf-unwind.c" -#undef main - #define main main_test_zlib # include "test-zlib.c" #undef main @@ -187,8 +179,7 @@ int main(int argc, char *argv[]) main_test_get_current_dir_name(); main_test_gettid(); main_test_glibc(); - main_test_dwarf(); - main_test_dwarf_getlocations(); + main_test_libdw(); main_test_eventfd(); main_test_libelf_getphdrnum(); main_test_libelf_gelf_getnote(); @@ -202,7 +193,6 @@ int main(int argc, char *argv[]) main_test_numa_num_possible_cpus(); main_test_timerfd(); main_test_stackprotector_all(); - main_test_libdw_dwarf_unwind(); main_test_zlib(); main_test_pthread_attr_setaffinity_np(); main_test_pthread_barrier(); diff --git a/tools/build/feature/test-dwarf.c b/tools/build/feature/test-dwarf.c deleted file mode 100644 index 8d474bd7371b..000000000000 --- a/tools/build/feature/test-dwarf.c +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <dwarf.h> -#include <elfutils/libdw.h> -#include <elfutils/version.h> - -int main(void) -{ - Dwarf *dbg = dwarf_begin(0, DWARF_C_READ); - - return (long)dbg; -} diff --git a/tools/build/feature/test-dwarf_getcfi.c b/tools/build/feature/test-dwarf_getcfi.c deleted file mode 100644 index 50e7d7cb7bdf..000000000000 --- a/tools/build/feature/test-dwarf_getcfi.c +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <stdio.h> -#include <elfutils/libdw.h> - -int main(void) -{ - Dwarf *dwarf = NULL; - return dwarf_getcfi(dwarf) == NULL; -} diff --git a/tools/build/feature/test-dwarf_getlocations.c b/tools/build/feature/test-dwarf_getlocations.c deleted file mode 100644 index 78fb4a1fa68c..000000000000 --- a/tools/build/feature/test-dwarf_getlocations.c +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <stdlib.h> -#include <elfutils/libdw.h> - -int main(void) -{ - Dwarf_Addr base, start, end; - Dwarf_Attribute attr; - Dwarf_Op *op; - size_t nops; - ptrdiff_t offset = 0; - return (int)dwarf_getlocations(&attr, offset, &base, &start, &end, &op, &nops); -} diff --git a/tools/build/feature/test-libdw-dwarf-unwind.c b/tools/build/feature/test-libdw-dwarf-unwind.c deleted file mode 100644 index ed03d9505609..000000000000 --- a/tools/build/feature/test-libdw-dwarf-unwind.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <elfutils/libdwfl.h> - -int main(void) -{ - /* - * This function is guarded via: __nonnull_attribute__ (1, 2). - * Passing '1' as arguments value. This code is never executed, - * only compiled. - */ - dwfl_thread_getframes((void *) 1, (void *) 1, NULL); - return 0; -} diff --git a/tools/build/feature/test-libdw.c b/tools/build/feature/test-libdw.c new file mode 100644 index 000000000000..2fb59479ab77 --- /dev/null +++ b/tools/build/feature/test-libdw.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdlib.h> +#include <dwarf.h> +#include <elfutils/libdw.h> +#include <elfutils/libdwfl.h> +#include <elfutils/version.h> + +int test_libdw(void) +{ + Dwarf *dbg = dwarf_begin(0, DWARF_C_READ); + + return (long)dbg; +} + +int test_libdw_unwind(void) +{ + /* + * This function is guarded via: __nonnull_attribute__ (1, 2). + * Passing '1' as arguments value. This code is never executed, + * only compiled. + */ + dwfl_thread_getframes((void *) 1, (void *) 1, NULL); + return 0; +} + +int test_libdw_getlocations(void) +{ + Dwarf_Addr base, start, end; + Dwarf_Attribute attr; + Dwarf_Op *op; + size_t nops; + ptrdiff_t offset = 0; + + return (int)dwarf_getlocations(&attr, offset, &base, &start, &end, &op, &nops); +} + +int test_libdw_getcfi(void) +{ + Dwarf *dwarf = NULL; + + return dwarf_getcfi(dwarf) == NULL; +} + +int test_elfutils(void) +{ + Dwarf_CFI *cfi = NULL; + + dwarf_cfi_end(cfi); + return 0; +} + +int main(void) +{ + return test_libdw() + test_libdw_unwind() + test_libdw_getlocations() + + test_libdw_getcfi() + test_elfutils(); +} diff --git a/tools/build/feature/test-libtraceevent.c b/tools/build/feature/test-libtraceevent.c index 416b11ffd4b4..804ad80dbbd9 100644 --- a/tools/build/feature/test-libtraceevent.c +++ b/tools/build/feature/test-libtraceevent.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include <traceevent/trace-seq.h> +#include <trace-seq.h> int main(void) { diff --git a/tools/lib/api/io.h b/tools/lib/api/io.h index d3eb04d1bc89..1731996b2c32 100644 --- a/tools/lib/api/io.h +++ b/tools/lib/api/io.h @@ -189,6 +189,7 @@ static inline ssize_t io__getdelim(struct io *io, char **line_out, size_t *line_ err_out: free(line); *line_out = NULL; + *line_len_out = 0; return -ENOMEM; } diff --git a/tools/lib/perf/Documentation/Makefile b/tools/lib/perf/Documentation/Makefile index 972754082a85..573ca5b27556 100644 --- a/tools/lib/perf/Documentation/Makefile +++ b/tools/lib/perf/Documentation/Makefile @@ -121,7 +121,7 @@ install-man: all $(INSTALL) -d -m 755 $(DESTDIR)$(man7dir); \ $(INSTALL) -m 644 $(MAN_7) $(DESTDIR)$(man7dir); -install-html: +install-html: $(MAN_HTML) $(call QUIET_INSTALL, html) \ $(INSTALL) -d -m 755 $(DESTDIR)$(htmldir); \ $(INSTALL) -m 644 $(MAN_HTML) $(DESTDIR)$(htmldir); \ diff --git a/tools/lib/perf/evsel.c b/tools/lib/perf/evsel.c index c07160953224..c475319e2e41 100644 --- a/tools/lib/perf/evsel.c +++ b/tools/lib/perf/evsel.c @@ -5,6 +5,7 @@ #include <perf/evsel.h> #include <perf/cpumap.h> #include <perf/threadmap.h> +#include <linux/hash.h> #include <linux/list.h> #include <internal/evsel.h> #include <linux/zalloc.h> @@ -23,6 +24,7 @@ void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, int idx) { INIT_LIST_HEAD(&evsel->node); + INIT_LIST_HEAD(&evsel->per_stream_periods); evsel->attr = *attr; evsel->idx = idx; evsel->leader = evsel; @@ -531,10 +533,56 @@ int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) void perf_evsel__free_id(struct perf_evsel *evsel) { + struct perf_sample_id_period *pos, *n; + xyarray__delete(evsel->sample_id); evsel->sample_id = NULL; zfree(&evsel->id); evsel->ids = 0; + + perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) { + list_del_init(&pos->node); + free(pos); + } +} + +bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel) +{ + return (evsel->attr.sample_type & PERF_SAMPLE_READ) && + (evsel->attr.sample_type & PERF_SAMPLE_TID) && + evsel->attr.inherit; +} + +u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread) +{ + struct hlist_head *head; + struct perf_sample_id_period *res; + int hash; + + if (!per_thread) + return &sid->period; + + hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS); + head = &sid->periods[hash]; + + hlist_for_each_entry(res, head, hnode) + if (res->tid == tid) + return &res->period; + + if (sid->evsel == NULL) + return NULL; + + res = zalloc(sizeof(struct perf_sample_id_period)); + if (res == NULL) + return NULL; + + INIT_LIST_HEAD(&res->node); + res->tid = tid; + + list_add_tail(&res->node, &sid->evsel->per_stream_periods); + hlist_add_head(&res->hnode, &sid->periods[hash]); + + return &res->period; } void perf_counts_values__scale(struct perf_counts_values *count, diff --git a/tools/lib/perf/include/internal/evsel.h b/tools/lib/perf/include/internal/evsel.h index 5cd220a61962..ea78defa77d0 100644 --- a/tools/lib/perf/include/internal/evsel.h +++ b/tools/lib/perf/include/internal/evsel.h @@ -11,6 +11,32 @@ struct perf_thread_map; struct xyarray; +/** + * The per-thread accumulated period storage node. + */ +struct perf_sample_id_period { + struct list_head node; + struct hlist_node hnode; + /* Holds total ID period value for PERF_SAMPLE_READ processing. */ + u64 period; + /* The TID that the values belongs to */ + u32 tid; +}; + +/** + * perf_evsel_for_each_per_thread_period_safe - safely iterate thru all the + * per_stream_periods + * @evlist:perf_evsel instance to iterate + * @item: struct perf_sample_id_period iterator + * @tmp: struct perf_sample_id_period temp iterator + */ +#define perf_evsel_for_each_per_thread_period_safe(evsel, tmp, item) \ + list_for_each_entry_safe(item, tmp, &(evsel)->per_stream_periods, node) + + +#define PERF_SAMPLE_ID__HLIST_BITS 4 +#define PERF_SAMPLE_ID__HLIST_SIZE (1 << PERF_SAMPLE_ID__HLIST_BITS) + /* * Per fd, to map back from PERF_SAMPLE_ID to evsel, only used when there are * more than one entry in the evlist. @@ -34,8 +60,32 @@ struct perf_sample_id { pid_t machine_pid; struct perf_cpu vcpu; - /* Holds total ID period value for PERF_SAMPLE_READ processing. */ - u64 period; + /* + * Per-thread, and global event counts are mutually exclusive: + * Whilst it is possible to combine events into a group with differing + * values of PERF_SAMPLE_READ, it is not valid to have inconsistent + * values for `inherit`. Therefore it is not possible to have a + * situation where a per-thread event is sampled as a global event; + * all !inherit groups are global, and all groups where the sampling + * event is inherit + PERF_SAMPLE_READ will be per-thread. Any event + * that is part of such a group that is inherit but not PERF_SAMPLE_READ + * will be read as per-thread. If such an event can also trigger a + * sample (such as with sample_period > 0) then it will not cause + * `read_format` to be included in its PERF_RECORD_SAMPLE, and + * therefore will not expose the per-thread group members as global. + */ + union { + /* + * Holds total ID period value for PERF_SAMPLE_READ processing + * (when period is not per-thread). + */ + u64 period; + /* + * Holds total ID period value for PERF_SAMPLE_READ processing + * (when period is per-thread). + */ + struct hlist_head periods[PERF_SAMPLE_ID__HLIST_SIZE]; + }; }; struct perf_evsel { @@ -58,6 +108,10 @@ struct perf_evsel { u32 ids; struct perf_evsel *leader; + /* For events where the read_format value is per-thread rather than + * global, stores the per-thread cumulative period */ + struct list_head per_stream_periods; + /* parse modifier helper */ int nr_members; /* @@ -88,4 +142,9 @@ int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter); int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads); void perf_evsel__free_id(struct perf_evsel *evsel); +bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel); + +u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, + bool per_thread); + #endif /* __LIBPERF_INTERNAL_EVSEL_H */ diff --git a/tools/lib/subcmd/run-command.c b/tools/lib/subcmd/run-command.c index 4e3a557a2f37..0a764c25c384 100644 --- a/tools/lib/subcmd/run-command.c +++ b/tools/lib/subcmd/run-command.c @@ -2,6 +2,7 @@ #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> +#include <ctype.h> #include <fcntl.h> #include <string.h> #include <linux/string.h> @@ -217,8 +218,40 @@ static int wait_or_whine(struct child_process *cmd, bool block) int check_if_command_finished(struct child_process *cmd) { +#ifdef __linux__ + char filename[FILENAME_MAX + 12]; + char status_line[256]; + FILE *status_file; + + /* + * Check by reading /proc/<pid>/status as calling waitpid causes + * stdout/stderr to be closed and data lost. + */ + sprintf(filename, "/proc/%d/status", cmd->pid); + status_file = fopen(filename, "r"); + if (status_file == NULL) { + /* Open failed assume finish_command was called. */ + return true; + } + while (fgets(status_line, sizeof(status_line), status_file) != NULL) { + char *p; + + if (strncmp(status_line, "State:", 6)) + continue; + + fclose(status_file); + p = status_line + 6; + while (isspace(*p)) + p++; + return *p == 'Z' ? 1 : 0; + } + /* Read failed assume finish_command was called. */ + fclose(status_file); + return 1; +#else wait_or_whine(cmd, /*block=*/false); return cmd->finished; +#endif } int finish_command(struct child_process *cmd) diff --git a/tools/lib/subcmd/subcmd-util.h b/tools/lib/subcmd/subcmd-util.h index dfac76e35ac7..c742b08815dc 100644 --- a/tools/lib/subcmd/subcmd-util.h +++ b/tools/lib/subcmd/subcmd-util.h @@ -20,8 +20,8 @@ static __noreturn inline void die(const char *err, ...) va_start(params, err); report(" Fatal: ", err, params); - exit(128); va_end(params); + exit(128); } #define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) diff --git a/tools/net/sunrpc/extract.sh b/tools/net/sunrpc/extract.sh new file mode 100755 index 000000000000..f944066f25bc --- /dev/null +++ b/tools/net/sunrpc/extract.sh @@ -0,0 +1,11 @@ +#! /bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Extract an RPC protocol specification from an RFC document. +# The version of this script comes from RFC 8166. +# +# Usage: +# $ extract.sh < rfcNNNN.txt > protocol.x +# + +grep '^ *///' | sed 's?^ */// ??' | sed 's?^ *///$??' diff --git a/tools/net/sunrpc/xdrgen/README b/tools/net/sunrpc/xdrgen/README index 92f7738ad50c..27218a78ab40 100644 --- a/tools/net/sunrpc/xdrgen/README +++ b/tools/net/sunrpc/xdrgen/README @@ -150,6 +150,23 @@ Pragma directives specify exceptions to the normal generation of encoding and decoding functions. Currently one directive is implemented: "public". +Pragma big_endian +------ ---------- + + pragma big_endian <enum> ; + +For variables that might contain only a small number values, it +is more efficient to avoid the byte-swap when encoding or decoding +on little-endian machines. Such is often the case with error status +codes. For example: + + pragma big_endian nfsstat3; + +In this case, when generating an XDR struct or union containing a +field of type "nfsstat3", xdrgen will make the type of that field +"__be32" instead of "enum nfsstat3". XDR unions then switch on the +non-byte-swapped value of that field. + Pragma exclude ------ ------- diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py index fd2457461274..b98574a36a4a 100644 --- a/tools/net/sunrpc/xdrgen/generators/__init__.py +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -111,3 +111,7 @@ class SourceGenerator: def emit_encoder(self, node: _XdrAst) -> None: """Emit one encoder function for this XDR type""" raise NotImplementedError("Encoder generation not supported") + + def emit_maxsize(self, node: _XdrAst) -> None: + """Emit one maxsize macro for this XDR type""" + raise NotImplementedError("Maxsize macro generation not supported") diff --git a/tools/net/sunrpc/xdrgen/generators/enum.py b/tools/net/sunrpc/xdrgen/generators/enum.py index 855e43f4ae38..e62f715d3996 100644 --- a/tools/net/sunrpc/xdrgen/generators/enum.py +++ b/tools/net/sunrpc/xdrgen/generators/enum.py @@ -4,7 +4,7 @@ """Generate code to handle XDR enum types""" from generators import SourceGenerator, create_jinja2_environment -from xdr_ast import _XdrEnum, public_apis +from xdr_ast import _XdrEnum, public_apis, big_endian, get_header_name class XdrEnumGenerator(SourceGenerator): @@ -18,7 +18,7 @@ class XdrEnumGenerator(SourceGenerator): def emit_declaration(self, node: _XdrEnum) -> None: """Emit one declaration pair for an XDR enum type""" if node.name in public_apis: - template = self.environment.get_template("declaration/close.j2") + template = self.environment.get_template("declaration/enum.j2") print(template.render(name=node.name)) def emit_definition(self, node: _XdrEnum) -> None: @@ -30,15 +30,35 @@ class XdrEnumGenerator(SourceGenerator): for enumerator in node.enumerators: print(template.render(name=enumerator.name, value=enumerator.value)) - template = self.environment.get_template("definition/close.j2") + if node.name in big_endian: + template = self.environment.get_template("definition/close_be.j2") + else: + template = self.environment.get_template("definition/close.j2") print(template.render(name=node.name)) def emit_decoder(self, node: _XdrEnum) -> None: """Emit one decoder function for an XDR enum type""" - template = self.environment.get_template("decoder/enum.j2") + if node.name in big_endian: + template = self.environment.get_template("decoder/enum_be.j2") + else: + template = self.environment.get_template("decoder/enum.j2") print(template.render(name=node.name)) def emit_encoder(self, node: _XdrEnum) -> None: """Emit one encoder function for an XDR enum type""" - template = self.environment.get_template("encoder/enum.j2") + if node.name in big_endian: + template = self.environment.get_template("encoder/enum_be.j2") + else: + template = self.environment.get_template("encoder/enum.j2") print(template.render(name=node.name)) + + def emit_maxsize(self, node: _XdrEnum) -> None: + """Emit one maxsize macro for an XDR enum type""" + macro_name = get_header_name().upper() + "_" + node.name + "_sz" + template = self.environment.get_template("maxsize/enum.j2") + print( + template.render( + macro=macro_name, + width=" + ".join(node.symbolic_width()), + ) + ) diff --git a/tools/net/sunrpc/xdrgen/generators/pointer.py b/tools/net/sunrpc/xdrgen/generators/pointer.py index b0b27f1819c8..6dbda60ad2db 100644 --- a/tools/net/sunrpc/xdrgen/generators/pointer.py +++ b/tools/net/sunrpc/xdrgen/generators/pointer.py @@ -8,11 +8,11 @@ from jinja2 import Environment from generators import SourceGenerator, kernel_c_type from generators import create_jinja2_environment, get_jinja2_template -from xdr_ast import _XdrBasic, _XdrVariableLengthString +from xdr_ast import _XdrBasic, _XdrString from xdr_ast import _XdrFixedLengthOpaque, _XdrVariableLengthOpaque from xdr_ast import _XdrFixedLengthArray, _XdrVariableLengthArray from xdr_ast import _XdrOptionalData, _XdrPointer, _XdrDeclaration -from xdr_ast import public_apis +from xdr_ast import public_apis, get_header_name def emit_pointer_declaration(environment: Environment, node: _XdrPointer) -> None: @@ -46,7 +46,7 @@ def emit_pointer_member_definition( elif isinstance(field, _XdrVariableLengthOpaque): template = get_jinja2_template(environment, "definition", field.template) print(template.render(name=field.name)) - elif isinstance(field, _XdrVariableLengthString): + elif isinstance(field, _XdrString): template = get_jinja2_template(environment, "definition", field.template) print(template.render(name=field.name)) elif isinstance(field, _XdrFixedLengthArray): @@ -119,7 +119,7 @@ def emit_pointer_member_decoder( maxsize=field.maxsize, ) ) - elif isinstance(field, _XdrVariableLengthString): + elif isinstance(field, _XdrString): template = get_jinja2_template(environment, "decoder", field.template) print( template.render( @@ -198,7 +198,7 @@ def emit_pointer_member_encoder( maxsize=field.maxsize, ) ) - elif isinstance(field, _XdrVariableLengthString): + elif isinstance(field, _XdrString): template = get_jinja2_template(environment, "encoder", field.template) print( template.render( @@ -247,6 +247,18 @@ def emit_pointer_encoder(environment: Environment, node: _XdrPointer) -> None: print(template.render()) +def emit_pointer_maxsize(environment: Environment, node: _XdrPointer) -> None: + """Emit one maxsize macro for an XDR pointer type""" + macro_name = get_header_name().upper() + "_" + node.name + "_sz" + template = get_jinja2_template(environment, "maxsize", "pointer") + print( + template.render( + macro=macro_name, + width=" + ".join(node.symbolic_width()), + ) + ) + + class XdrPointerGenerator(SourceGenerator): """Generate source code for XDR pointer""" @@ -270,3 +282,7 @@ class XdrPointerGenerator(SourceGenerator): def emit_encoder(self, node: _XdrPointer) -> None: """Emit one encoder function for an XDR pointer type""" emit_pointer_encoder(self.environment, node) + + def emit_maxsize(self, node: _XdrPointer) -> None: + """Emit one maxsize macro for an XDR pointer type""" + emit_pointer_maxsize(self.environment, node) diff --git a/tools/net/sunrpc/xdrgen/generators/struct.py b/tools/net/sunrpc/xdrgen/generators/struct.py index b694cd470829..64911de46f62 100644 --- a/tools/net/sunrpc/xdrgen/generators/struct.py +++ b/tools/net/sunrpc/xdrgen/generators/struct.py @@ -8,11 +8,11 @@ from jinja2 import Environment from generators import SourceGenerator, kernel_c_type from generators import create_jinja2_environment, get_jinja2_template -from xdr_ast import _XdrBasic, _XdrVariableLengthString +from xdr_ast import _XdrBasic, _XdrString from xdr_ast import _XdrFixedLengthOpaque, _XdrVariableLengthOpaque from xdr_ast import _XdrFixedLengthArray, _XdrVariableLengthArray from xdr_ast import _XdrOptionalData, _XdrStruct, _XdrDeclaration -from xdr_ast import public_apis +from xdr_ast import public_apis, get_header_name def emit_struct_declaration(environment: Environment, node: _XdrStruct) -> None: @@ -46,7 +46,7 @@ def emit_struct_member_definition( elif isinstance(field, _XdrVariableLengthOpaque): template = get_jinja2_template(environment, "definition", field.template) print(template.render(name=field.name)) - elif isinstance(field, _XdrVariableLengthString): + elif isinstance(field, _XdrString): template = get_jinja2_template(environment, "definition", field.template) print(template.render(name=field.name)) elif isinstance(field, _XdrFixedLengthArray): @@ -119,7 +119,7 @@ def emit_struct_member_decoder( maxsize=field.maxsize, ) ) - elif isinstance(field, _XdrVariableLengthString): + elif isinstance(field, _XdrString): template = get_jinja2_template(environment, "decoder", field.template) print( template.render( @@ -198,7 +198,7 @@ def emit_struct_member_encoder( maxsize=field.maxsize, ) ) - elif isinstance(field, _XdrVariableLengthString): + elif isinstance(field, _XdrString): template = get_jinja2_template(environment, "encoder", field.template) print( template.render( @@ -247,6 +247,18 @@ def emit_struct_encoder(environment: Environment, node: _XdrStruct) -> None: print(template.render()) +def emit_struct_maxsize(environment: Environment, node: _XdrStruct) -> None: + """Emit one maxsize macro for an XDR struct type""" + macro_name = get_header_name().upper() + "_" + node.name + "_sz" + template = get_jinja2_template(environment, "maxsize", "struct") + print( + template.render( + macro=macro_name, + width=" + ".join(node.symbolic_width()), + ) + ) + + class XdrStructGenerator(SourceGenerator): """Generate source code for XDR structs""" @@ -270,3 +282,7 @@ class XdrStructGenerator(SourceGenerator): def emit_encoder(self, node: _XdrStruct) -> None: """Emit one encoder function for an XDR struct type""" emit_struct_encoder(self.environment, node) + + def emit_maxsize(self, node: _XdrStruct) -> None: + """Emit one maxsize macro for an XDR struct type""" + emit_struct_maxsize(self.environment, node) diff --git a/tools/net/sunrpc/xdrgen/generators/typedef.py b/tools/net/sunrpc/xdrgen/generators/typedef.py index 85a1b2303333..fab72e9d6915 100644 --- a/tools/net/sunrpc/xdrgen/generators/typedef.py +++ b/tools/net/sunrpc/xdrgen/generators/typedef.py @@ -8,11 +8,11 @@ from jinja2 import Environment from generators import SourceGenerator, kernel_c_type from generators import create_jinja2_environment, get_jinja2_template -from xdr_ast import _XdrBasic, _XdrTypedef, _XdrVariableLengthString +from xdr_ast import _XdrBasic, _XdrTypedef, _XdrString from xdr_ast import _XdrFixedLengthOpaque, _XdrVariableLengthOpaque from xdr_ast import _XdrFixedLengthArray, _XdrVariableLengthArray from xdr_ast import _XdrOptionalData, _XdrVoid, _XdrDeclaration -from xdr_ast import public_apis +from xdr_ast import public_apis, get_header_name def emit_typedef_declaration(environment: Environment, node: _XdrDeclaration) -> None: @@ -28,7 +28,7 @@ def emit_typedef_declaration(environment: Environment, node: _XdrDeclaration) -> classifier=node.spec.c_classifier, ) ) - elif isinstance(node, _XdrVariableLengthString): + elif isinstance(node, _XdrString): template = get_jinja2_template(environment, "declaration", node.template) print(template.render(name=node.name)) elif isinstance(node, _XdrFixedLengthOpaque): @@ -74,7 +74,7 @@ def emit_type_definition(environment: Environment, node: _XdrDeclaration) -> Non classifier=node.spec.c_classifier, ) ) - elif isinstance(node, _XdrVariableLengthString): + elif isinstance(node, _XdrString): template = get_jinja2_template(environment, "definition", node.template) print(template.render(name=node.name)) elif isinstance(node, _XdrFixedLengthOpaque): @@ -119,7 +119,7 @@ def emit_typedef_decoder(environment: Environment, node: _XdrDeclaration) -> Non type=node.spec.type_name, ) ) - elif isinstance(node, _XdrVariableLengthString): + elif isinstance(node, _XdrString): template = get_jinja2_template(environment, "decoder", node.template) print( template.render( @@ -180,7 +180,7 @@ def emit_typedef_encoder(environment: Environment, node: _XdrDeclaration) -> Non type=node.spec.type_name, ) ) - elif isinstance(node, _XdrVariableLengthString): + elif isinstance(node, _XdrString): template = get_jinja2_template(environment, "encoder", node.template) print( template.render( @@ -230,6 +230,18 @@ def emit_typedef_encoder(environment: Environment, node: _XdrDeclaration) -> Non raise NotImplementedError("typedef: type not recognized") +def emit_typedef_maxsize(environment: Environment, node: _XdrDeclaration) -> None: + """Emit a maxsize macro for an XDR typedef""" + macro_name = get_header_name().upper() + "_" + node.name + "_sz" + template = get_jinja2_template(environment, "maxsize", node.template) + print( + template.render( + macro=macro_name, + width=" + ".join(node.symbolic_width()), + ) + ) + + class XdrTypedefGenerator(SourceGenerator): """Generate source code for XDR typedefs""" @@ -253,3 +265,7 @@ class XdrTypedefGenerator(SourceGenerator): def emit_encoder(self, node: _XdrTypedef) -> None: """Emit one encoder function for an XDR typedef""" emit_typedef_encoder(self.environment, node.declaration) + + def emit_maxsize(self, node: _XdrTypedef) -> None: + """Emit one maxsize macro for an XDR typedef""" + emit_typedef_maxsize(self.environment, node.declaration) diff --git a/tools/net/sunrpc/xdrgen/generators/union.py b/tools/net/sunrpc/xdrgen/generators/union.py index 7974967bbb9f..2cca00e279cd 100644 --- a/tools/net/sunrpc/xdrgen/generators/union.py +++ b/tools/net/sunrpc/xdrgen/generators/union.py @@ -8,8 +8,8 @@ from jinja2 import Environment from generators import SourceGenerator from generators import create_jinja2_environment, get_jinja2_template -from xdr_ast import _XdrBasic, _XdrUnion, _XdrVoid -from xdr_ast import _XdrDeclaration, _XdrCaseSpec, public_apis +from xdr_ast import _XdrBasic, _XdrUnion, _XdrVoid, get_header_name +from xdr_ast import _XdrDeclaration, _XdrCaseSpec, public_apis, big_endian def emit_union_declaration(environment: Environment, node: _XdrUnion) -> None: @@ -77,13 +77,18 @@ def emit_union_switch_spec_decoder( print(template.render(name=node.name, type=node.spec.type_name)) -def emit_union_case_spec_decoder(environment: Environment, node: _XdrCaseSpec) -> None: +def emit_union_case_spec_decoder( + environment: Environment, node: _XdrCaseSpec, big_endian_discriminant: bool +) -> None: """Emit decoder functions for an XDR union's case arm""" if isinstance(node.arm, _XdrVoid): return - template = get_jinja2_template(environment, "decoder", "case_spec") + if big_endian_discriminant: + template = get_jinja2_template(environment, "decoder", "case_spec_be") + else: + template = get_jinja2_template(environment, "decoder", "case_spec") for case in node.values: print(template.render(case=case)) @@ -136,7 +141,11 @@ def emit_union_decoder(environment: Environment, node: _XdrUnion) -> None: emit_union_switch_spec_decoder(environment, node.discriminant) for case in node.cases: - emit_union_case_spec_decoder(environment, case) + emit_union_case_spec_decoder( + environment, + case, + node.discriminant.spec.type_name in big_endian, + ) emit_union_default_spec_decoder(environment, node) @@ -153,17 +162,21 @@ def emit_union_switch_spec_encoder( print(template.render(name=node.name, type=node.spec.type_name)) -def emit_union_case_spec_encoder(environment: Environment, node: _XdrCaseSpec) -> None: +def emit_union_case_spec_encoder( + environment: Environment, node: _XdrCaseSpec, big_endian_discriminant: bool +) -> None: """Emit encoder functions for an XDR union's case arm""" if isinstance(node.arm, _XdrVoid): return - template = get_jinja2_template(environment, "encoder", "case_spec") + if big_endian_discriminant: + template = get_jinja2_template(environment, "encoder", "case_spec_be") + else: + template = get_jinja2_template(environment, "encoder", "case_spec") for case in node.values: print(template.render(case=case)) - assert isinstance(node.arm, _XdrBasic) template = get_jinja2_template(environment, "encoder", node.arm.template) print( template.render( @@ -192,7 +205,6 @@ def emit_union_default_spec_encoder(environment: Environment, node: _XdrUnion) - print(template.render()) return - assert isinstance(default_case.arm, _XdrBasic) template = get_jinja2_template(environment, "encoder", default_case.arm.template) print( template.render( @@ -210,7 +222,11 @@ def emit_union_encoder(environment, node: _XdrUnion) -> None: emit_union_switch_spec_encoder(environment, node.discriminant) for case in node.cases: - emit_union_case_spec_encoder(environment, case) + emit_union_case_spec_encoder( + environment, + case, + node.discriminant.spec.type_name in big_endian, + ) emit_union_default_spec_encoder(environment, node) @@ -218,6 +234,18 @@ def emit_union_encoder(environment, node: _XdrUnion) -> None: print(template.render()) +def emit_union_maxsize(environment: Environment, node: _XdrUnion) -> None: + """Emit one maxsize macro for an XDR union type""" + macro_name = get_header_name().upper() + "_" + node.name + "_sz" + template = get_jinja2_template(environment, "maxsize", "union") + print( + template.render( + macro=macro_name, + width=" + ".join(node.symbolic_width()), + ) + ) + + class XdrUnionGenerator(SourceGenerator): """Generate source code for XDR unions""" @@ -241,3 +269,7 @@ class XdrUnionGenerator(SourceGenerator): def emit_encoder(self, node: _XdrUnion) -> None: """Emit one encoder function for an XDR union""" emit_union_encoder(self.environment, node) + + def emit_maxsize(self, node: _XdrUnion) -> None: + """Emit one maxsize macro for an XDR union""" + emit_union_maxsize(self.environment, node) diff --git a/tools/net/sunrpc/xdrgen/grammars/xdr.lark b/tools/net/sunrpc/xdrgen/grammars/xdr.lark index f3c4552e548d..7c2c1b8c86d1 100644 --- a/tools/net/sunrpc/xdrgen/grammars/xdr.lark +++ b/tools/net/sunrpc/xdrgen/grammars/xdr.lark @@ -3,7 +3,7 @@ declaration : "opaque" identifier "[" value "]" -> fixed_length_opaque | "opaque" identifier "<" [ value ] ">" -> variable_length_opaque - | "string" identifier "<" [ value ] ">" -> variable_length_string + | "string" identifier "<" [ value ] ">" -> string | type_specifier identifier "[" value "]" -> fixed_length_array | type_specifier identifier "<" [ value ] ">" -> variable_length_array | type_specifier "*" identifier -> optional_data @@ -87,12 +87,14 @@ procedure_def : type_specifier identifier "(" type_specifier ")" "=" c pragma_def : "pragma" directive identifier [ identifier ] ";" -directive : exclude_directive +directive : big_endian_directive + | exclude_directive | header_directive | pages_directive | public_directive | skip_directive +big_endian_directive : "big_endian" exclude_directive : "exclude" header_directive : "header" pages_directive : "pages" diff --git a/tools/net/sunrpc/xdrgen/subcmds/definitions.py b/tools/net/sunrpc/xdrgen/subcmds/definitions.py index 5cd13d53221f..c956e27f37c0 100644 --- a/tools/net/sunrpc/xdrgen/subcmds/definitions.py +++ b/tools/net/sunrpc/xdrgen/subcmds/definitions.py @@ -28,9 +28,7 @@ from xdr_parse import xdr_parser, set_xdr_annotate logger.setLevel(logging.INFO) -def emit_header_definitions( - root: Specification, language: str, peer: str -) -> None: +def emit_header_definitions(root: Specification, language: str, peer: str) -> None: """Emit header definitions""" for definition in root.definitions: if isinstance(definition.value, _XdrConstant): @@ -52,6 +50,25 @@ def emit_header_definitions( gen.emit_definition(definition.value) +def emit_header_maxsize(root: Specification, language: str, peer: str) -> None: + """Emit header maxsize macros""" + print("") + for definition in root.definitions: + if isinstance(definition.value, _XdrEnum): + gen = XdrEnumGenerator(language, peer) + elif isinstance(definition.value, _XdrPointer): + gen = XdrPointerGenerator(language, peer) + elif isinstance(definition.value, _XdrTypedef): + gen = XdrTypedefGenerator(language, peer) + elif isinstance(definition.value, _XdrStruct): + gen = XdrStructGenerator(language, peer) + elif isinstance(definition.value, _XdrUnion): + gen = XdrUnionGenerator(language, peer) + else: + continue + gen.emit_maxsize(definition.value) + + def handle_parse_error(e: UnexpectedInput) -> bool: """Simple parse error reporting, no recovery attempted""" print(e) @@ -71,6 +88,7 @@ def subcmd(args: Namespace) -> int: gen.emit_definition(args.filename, ast) emit_header_definitions(ast, args.language, args.peer) + emit_header_maxsize(ast, args.language, args.peer) gen = XdrHeaderBottomGenerator(args.language, args.peer) gen.emit_definition(args.filename, ast) diff --git a/tools/net/sunrpc/xdrgen/subcmds/source.py b/tools/net/sunrpc/xdrgen/subcmds/source.py index 00c04ad15b89..2024954748f0 100644 --- a/tools/net/sunrpc/xdrgen/subcmds/source.py +++ b/tools/net/sunrpc/xdrgen/subcmds/source.py @@ -83,8 +83,7 @@ def generate_client_source(filename: str, root: Specification, language: str) -> gen = XdrSourceTopGenerator(language, "client") gen.emit_source(filename, root) - # cel: todo: client needs XDR size macros - + print("") for definition in root.definitions: emit_source_encoder(definition.value, language, "client") for definition in root.definitions: diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 deleted file mode 100644 index ab1e576c9531..000000000000 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/close.j2 +++ /dev/null @@ -1,4 +0,0 @@ -{# SPDX-License-Identifier: GPL-2.0 #} - -bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} *ptr); -bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 new file mode 100644 index 000000000000..d1405c7c5354 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 index 341d829afeda..6482984f1cb7 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum.j2 @@ -8,7 +8,7 @@ bool {% else %} static bool __maybe_unused {% endif %} -xdrgen_decode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} *ptr) +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr) { u32 val; diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum_be.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum_be.j2 new file mode 100644 index 000000000000..44c391c10b42 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/decoder/enum_be.j2 @@ -0,0 +1,14 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* enum {{ name }} (big-endian) */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr) +{ + return xdr_stream_decode_be32(xdr, ptr) == 0; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 index 9e62344a976a..a07586cbee17 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 @@ -1,2 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} }; +typedef enum {{ name }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 new file mode 100644 index 000000000000..2c18948bddf7 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +}; +typedef __be32 {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 index bd0a770e50f2..67245b9a914d 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum.j2 @@ -8,7 +8,7 @@ bool {% else %} static bool __maybe_unused {% endif %} -xdrgen_encode_{{ name }}(struct xdr_stream *xdr, enum {{ name }} value) +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value) { return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; } diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum_be.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum_be.j2 new file mode 100644 index 000000000000..fbbcc45948d6 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/encoder/enum_be.j2 @@ -0,0 +1,14 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +{% if annotate %} +/* enum {{ name }} (big-endian) */ +{% endif %} +{% if name in public_apis %} +bool +{% else %} +static bool __maybe_unused +{% endif %} +xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value) +{ + return xdr_stream_encode_be32(xdr, value) == XDR_UNIT; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/maxsize/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/maxsize/enum.j2 new file mode 100644 index 000000000000..45c1d4c21b22 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/maxsize/enum.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/string.j2 index 12d20b143b43..12d20b143b43 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/string.j2 index 2de2feec77db..2de2feec77db 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/definition/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 index cf65b71eaef3..cf65b71eaef3 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/maxsize/pointer.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/maxsize/pointer.j2 new file mode 100644 index 000000000000..9f3bfb47d2f4 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/maxsize/pointer.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} \ + ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 index d304eccb5c40..aa9940e322db 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/result.j2 @@ -13,10 +13,6 @@ static int {{ program }}_xdr_dec_{{ result }}(struct rpc_rqst *req, if (!xdrgen_decode_{{ result }}(xdr, result)) return -EIO; - if (result->stat != nfs_ok) { - trace_nfs_xdr_status(xdr, (int)result->stat); - return {{ program }}_stat_to_errno(result->stat); - } {% endif %} return 0; } diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 index e3a802cbc4d7..c5518c519854 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 @@ -3,6 +3,11 @@ // XDR specification file: {{ filename }} // XDR specification modification time: {{ mtime }} -#include <linux/sunrpc/xprt.h> +#include <linux/types.h> -#include "{{ program }}xdr_gen.h" +#include <linux/sunrpc/xdr.h> +#include <linux/sunrpc/xdrgen/_defs.h> +#include <linux/sunrpc/xdrgen/_builtins.h> +#include <linux/sunrpc/xdrgen/nlm4.h> + +#include <linux/sunrpc/clnt.h> diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/string.j2 index 12d20b143b43..12d20b143b43 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/string.j2 index 2de2feec77db..2de2feec77db 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/definition/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/definition/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 index cf65b71eaef3..cf65b71eaef3 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/maxsize/struct.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/maxsize/struct.j2 new file mode 100644 index 000000000000..9f3bfb47d2f4 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/maxsize/struct.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} \ + ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/string.j2 index 3fe3ddd9f359..3fe3ddd9f359 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/declaration/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/string.j2 index 56c5a17d6a70..56c5a17d6a70 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/string.j2 index c03c2df8e625..c03c2df8e625 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/definition/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/string.j2 index 3d490ff180d0..3d490ff180d0 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/basic.j2 new file mode 100644 index 000000000000..9f3bfb47d2f4 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/basic.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} \ + ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/fixed_length_opaque.j2 new file mode 100644 index 000000000000..45c1d4c21b22 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/fixed_length_opaque.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/string.j2 new file mode 100644 index 000000000000..45c1d4c21b22 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/string.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/variable_length_array.j2 new file mode 100644 index 000000000000..45c1d4c21b22 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/variable_length_array.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/variable_length_opaque.j2 new file mode 100644 index 000000000000..45c1d4c21b22 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/maxsize/variable_length_opaque.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec_be.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec_be.j2 new file mode 100644 index 000000000000..917f3a1c4588 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/case_spec_be.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + case __constant_cpu_to_be32({{ case }}): diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_string.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/string.j2 index 83b6e5a14e7f..83b6e5a14e7f 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/variable_length_string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/string.j2 diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec_be.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec_be.j2 new file mode 100644 index 000000000000..917f3a1c4588 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/case_spec_be.j2 @@ -0,0 +1,2 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + case __constant_cpu_to_be32({{ case }}): diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/maxsize/union.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/maxsize/union.j2 new file mode 100644 index 000000000000..9f3bfb47d2f4 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/maxsize/union.j2 @@ -0,0 +1,3 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +#define {{ '{:<31}'.format(macro) }} \ + ({{ width }}) diff --git a/tools/net/sunrpc/xdrgen/xdr_ast.py b/tools/net/sunrpc/xdrgen/xdr_ast.py index dbd3fcf9c957..5233e73c7046 100644 --- a/tools/net/sunrpc/xdrgen/xdr_ast.py +++ b/tools/net/sunrpc/xdrgen/xdr_ast.py @@ -12,13 +12,50 @@ from lark.tree import Meta this_module = sys.modules[__name__] +big_endian = [] excluded_apis = [] header_name = "none" public_apis = [] -enums = set() structs = set() pass_by_reference = set() +constants = {} + + +def xdr_quadlen(val: str) -> int: + """Return integer XDR width of an XDR type""" + if val in constants: + octets = constants[val] + else: + octets = int(val) + return int((octets + 3) / 4) + + +symbolic_widths = { + "void": ["XDR_void"], + "bool": ["XDR_bool"], + "int": ["XDR_int"], + "unsigned_int": ["XDR_unsigned_int"], + "long": ["XDR_long"], + "unsigned_long": ["XDR_unsigned_long"], + "hyper": ["XDR_hyper"], + "unsigned_hyper": ["XDR_unsigned_hyper"], +} + +# Numeric XDR widths are tracked in a dictionary that is keyed +# by type_name because sometimes a caller has nothing more than +# the type_name to use to figure out the numeric width. +max_widths = { + "void": 0, + "bool": 1, + "int": 1, + "unsigned_int": 1, + "long": 1, + "unsigned_long": 1, + "hyper": 2, + "unsigned_hyper": 2, +} + @dataclass class _XdrAst(ast_utils.Ast): @@ -51,18 +88,31 @@ class _XdrTypeSpecifier(_XdrAst): """Corresponds to 'type_specifier' in the XDR language grammar""" type_name: str - c_classifier: str + c_classifier: str = "" @dataclass class _XdrDefinedType(_XdrTypeSpecifier): """Corresponds to a type defined by the input specification""" + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return [get_header_name().upper() + "_" + self.type_name + "_sz"] + + def __post_init__(self): + if self.type_name in structs: + self.c_classifier = "struct " + symbolic_widths[self.type_name] = self.symbolic_width() + @dataclass class _XdrBuiltInType(_XdrTypeSpecifier): """Corresponds to a built-in XDR type""" + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return symbolic_widths[self.type_name] + @dataclass class _XdrDeclaration(_XdrAst): @@ -77,6 +127,18 @@ class _XdrFixedLengthOpaque(_XdrDeclaration): size: str template: str = "fixed_length_opaque" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return xdr_quadlen(self.size) + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return ["XDR_QUADLEN(" + self.size + ")"] + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrVariableLengthOpaque(_XdrDeclaration): @@ -86,14 +148,44 @@ class _XdrVariableLengthOpaque(_XdrDeclaration): maxsize: str template: str = "variable_length_opaque" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return 1 + xdr_quadlen(self.maxsize) + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + widths = ["XDR_unsigned_int"] + if self.maxsize != "0": + widths.append("XDR_QUADLEN(" + self.maxsize + ")") + return widths + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass -class _XdrVariableLengthString(_XdrDeclaration): +class _XdrString(_XdrDeclaration): """A (NUL-terminated) variable-length string declaration""" name: str maxsize: str - template: str = "variable_length_string" + template: str = "string" + + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return 1 + xdr_quadlen(self.maxsize) + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + widths = ["XDR_unsigned_int"] + if self.maxsize != "0": + widths.append("XDR_QUADLEN(" + self.maxsize + ")") + return widths + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() @dataclass @@ -105,6 +197,19 @@ class _XdrFixedLengthArray(_XdrDeclaration): size: str template: str = "fixed_length_array" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return xdr_quadlen(self.size) * max_widths[self.spec.type_name] + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + item_width = " + ".join(symbolic_widths[self.spec.type_name]) + return ["(" + self.size + " * (" + item_width + "))"] + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrVariableLengthArray(_XdrDeclaration): @@ -115,6 +220,22 @@ class _XdrVariableLengthArray(_XdrDeclaration): maxsize: str template: str = "variable_length_array" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return 1 + (xdr_quadlen(self.maxsize) * max_widths[self.spec.type_name]) + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + widths = ["XDR_unsigned_int"] + if self.maxsize != "0": + item_width = " + ".join(symbolic_widths[self.spec.type_name]) + widths.append("(" + self.maxsize + " * (" + item_width + "))") + return widths + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrOptionalData(_XdrDeclaration): @@ -124,6 +245,20 @@ class _XdrOptionalData(_XdrDeclaration): spec: _XdrTypeSpecifier template: str = "optional_data" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return 1 + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return ["XDR_bool"] + + def __post_init__(self): + structs.add(self.name) + pass_by_reference.add(self.name) + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrBasic(_XdrDeclaration): @@ -133,13 +268,34 @@ class _XdrBasic(_XdrDeclaration): spec: _XdrTypeSpecifier template: str = "basic" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return max_widths[self.spec.type_name] + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return symbolic_widths[self.spec.type_name] + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrVoid(_XdrDeclaration): """A void declaration""" + name: str = "void" template: str = "void" + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return 0 + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return [] + @dataclass class _XdrConstant(_XdrAst): @@ -148,6 +304,10 @@ class _XdrConstant(_XdrAst): name: str value: str + def __post_init__(self): + if self.value not in constants: + constants[self.name] = int(self.value, 0) + @dataclass class _XdrEnumerator(_XdrAst): @@ -156,6 +316,10 @@ class _XdrEnumerator(_XdrAst): name: str value: str + def __post_init__(self): + if self.value not in constants: + constants[self.name] = int(self.value, 0) + @dataclass class _XdrEnum(_XdrAst): @@ -166,6 +330,18 @@ class _XdrEnum(_XdrAst): maximum: int enumerators: List[_XdrEnumerator] + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return 1 + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return ["XDR_int"] + + def __post_init__(self): + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrStruct(_XdrAst): @@ -174,6 +350,26 @@ class _XdrStruct(_XdrAst): name: str fields: List[_XdrDeclaration] + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + width = 0 + for field in self.fields: + width += field.max_width() + return width + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + widths = [] + for field in self.fields: + widths += field.symbolic_width() + return widths + + def __post_init__(self): + structs.add(self.name) + pass_by_reference.add(self.name) + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrPointer(_XdrAst): @@ -182,6 +378,27 @@ class _XdrPointer(_XdrAst): name: str fields: List[_XdrDeclaration] + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + width = 1 + for field in self.fields[0:-1]: + width += field.max_width() + return width + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + widths = [] + widths += ["XDR_bool"] + for field in self.fields[0:-1]: + widths += field.symbolic_width() + return widths + + def __post_init__(self): + structs.add(self.name) + pass_by_reference.add(self.name) + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _XdrTypedef(_XdrAst): @@ -189,6 +406,23 @@ class _XdrTypedef(_XdrAst): declaration: _XdrDeclaration + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + return self.declaration.max_width() + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + return self.declaration.symbolic_width() + + def __post_init__(self): + if isinstance(self.declaration, _XdrBasic): + new_type = self.declaration + if isinstance(new_type.spec, _XdrDefinedType): + if new_type.spec.type_name in pass_by_reference: + pass_by_reference.add(new_type.name) + max_widths[new_type.name] = self.max_width() + symbolic_widths[new_type.name] = self.symbolic_width() + @dataclass class _XdrCaseSpec(_XdrAst): @@ -216,6 +450,36 @@ class _XdrUnion(_XdrAst): cases: List[_XdrCaseSpec] default: _XdrDeclaration + def max_width(self) -> int: + """Return width of type in XDR_UNITS""" + max_width = 0 + for case in self.cases: + if case.arm.max_width() > max_width: + max_width = case.arm.max_width() + if self.default: + if self.default.arm.max_width() > max_width: + max_width = self.default.arm.max_width() + return 1 + max_width + + def symbolic_width(self) -> List: + """Return list containing XDR width of type's components""" + max_width = 0 + for case in self.cases: + if case.arm.max_width() > max_width: + max_width = case.arm.max_width() + width = case.arm.symbolic_width() + if self.default: + if self.default.arm.max_width() > max_width: + max_width = self.default.arm.max_width() + width = self.default.arm.symbolic_width() + return symbolic_widths[self.discriminant.name] + width + + def __post_init__(self): + structs.add(self.name) + pass_by_reference.add(self.name) + max_widths[self.name] = self.max_width() + symbolic_widths[self.name] = self.symbolic_width() + @dataclass class _RpcProcedure(_XdrAst): @@ -290,24 +554,13 @@ class ParseToAst(Transformer): return _XdrConstantValue(value) def type_specifier(self, children): - """Instantiate one type_specifier object""" - c_classifier = "" + """Instantiate one _XdrTypeSpecifier object""" if isinstance(children[0], _XdrIdentifier): name = children[0].symbol - if name in enums: - c_classifier = "enum " - if name in structs: - c_classifier = "struct " - return _XdrDefinedType( - type_name=name, - c_classifier=c_classifier, - ) - - token = children[0].data - return _XdrBuiltInType( - type_name=token.value, - c_classifier=c_classifier, - ) + return _XdrDefinedType(type_name=name) + + name = children[0].data.value + return _XdrBuiltInType(type_name=name) def constant_def(self, children): """Instantiate one _XdrConstant object""" @@ -320,7 +573,6 @@ class ParseToAst(Transformer): def enum(self, children): """Instantiate one _XdrEnum object""" enum_name = children[0].symbol - enums.add(enum_name) i = 0 enumerators = [] @@ -350,15 +602,15 @@ class ParseToAst(Transformer): return _XdrVariableLengthOpaque(name, maxsize) - def variable_length_string(self, children): - """Instantiate one _XdrVariableLengthString declaration object""" + def string(self, children): + """Instantiate one _XdrString declaration object""" name = children[0].symbol if children[1] is not None: maxsize = children[1].value else: maxsize = "0" - return _XdrVariableLengthString(name, maxsize) + return _XdrString(name, maxsize) def fixed_length_array(self, children): """Instantiate one _XdrFixedLengthArray declaration object""" @@ -383,8 +635,6 @@ class ParseToAst(Transformer): """Instantiate one _XdrOptionalData declaration object""" spec = children[0] name = children[1].symbol - structs.add(name) - pass_by_reference.add(name) return _XdrOptionalData(name, spec) @@ -403,8 +653,6 @@ class ParseToAst(Transformer): def struct(self, children): """Instantiate one _XdrStruct object""" name = children[0].symbol - structs.add(name) - pass_by_reference.add(name) fields = children[1].children last_field = fields[-1] @@ -419,11 +667,6 @@ class ParseToAst(Transformer): def typedef(self, children): """Instantiate one _XdrTypedef object""" new_type = children[0] - if isinstance(new_type, _XdrBasic) and isinstance( - new_type.spec, _XdrDefinedType - ): - if new_type.spec.type_name in pass_by_reference: - pass_by_reference.add(new_type.name) return _XdrTypedef(new_type) @@ -445,8 +688,6 @@ class ParseToAst(Transformer): def union(self, children): """Instantiate one _XdrUnion object""" name = children[0].symbol - structs.add(name) - pass_by_reference.add(name) body = children[1] discriminant = body.children[0].children[0] @@ -484,6 +725,8 @@ class ParseToAst(Transformer): """Instantiate one _Pragma object""" directive = children[0].children[0].data match directive: + case "big_endian_directive": + big_endian.append(children[1].symbol) case "exclude_directive": excluded_apis.append(children[1].symbol) case "header_directive": diff --git a/tools/net/sunrpc/xdrgen/xdrgen b/tools/net/sunrpc/xdrgen/xdrgen index 95f303b2861b..43762be39252 100755 --- a/tools/net/sunrpc/xdrgen/xdrgen +++ b/tools/net/sunrpc/xdrgen/xdrgen @@ -128,5 +128,7 @@ There is NO WARRANTY, to the extent permitted by law.""", try: if __name__ == "__main__": sys.exit(main()) -except (SystemExit, KeyboardInterrupt, BrokenPipeError): +except SystemExit: + sys.exit(0) +except (KeyboardInterrupt, BrokenPipeError): sys.exit(1) diff --git a/tools/pci/pcitest.c b/tools/pci/pcitest.c index 470258009ddc..7b530d838d40 100644 --- a/tools/pci/pcitest.c +++ b/tools/pci/pcitest.c @@ -95,7 +95,7 @@ static int run_test(struct pci_test *test) if (test->msinum > 0 && test->msinum <= 32) { ret = ioctl(fd, PCITEST_MSI, test->msinum); - fprintf(stdout, "MSI%d:\t\t", test->msinum); + fprintf(stdout, "MSI%u:\t\t", test->msinum); if (ret < 0) fprintf(stdout, "TEST FAILED\n"); else @@ -104,7 +104,7 @@ static int run_test(struct pci_test *test) if (test->msixnum > 0 && test->msixnum <= 2048) { ret = ioctl(fd, PCITEST_MSIX, test->msixnum); - fprintf(stdout, "MSI-X%d:\t\t", test->msixnum); + fprintf(stdout, "MSI-X%u:\t\t", test->msixnum); if (ret < 0) fprintf(stdout, "TEST FAILED\n"); else @@ -116,7 +116,7 @@ static int run_test(struct pci_test *test) if (test->use_dma) param.flags = PCITEST_FLAGS_USE_DMA; ret = ioctl(fd, PCITEST_WRITE, ¶m); - fprintf(stdout, "WRITE (%7ld bytes):\t\t", test->size); + fprintf(stdout, "WRITE (%7lu bytes):\t\t", test->size); if (ret < 0) fprintf(stdout, "TEST FAILED\n"); else @@ -128,7 +128,7 @@ static int run_test(struct pci_test *test) if (test->use_dma) param.flags = PCITEST_FLAGS_USE_DMA; ret = ioctl(fd, PCITEST_READ, ¶m); - fprintf(stdout, "READ (%7ld bytes):\t\t", test->size); + fprintf(stdout, "READ (%7lu bytes):\t\t", test->size); if (ret < 0) fprintf(stdout, "TEST FAILED\n"); else @@ -140,7 +140,7 @@ static int run_test(struct pci_test *test) if (test->use_dma) param.flags = PCITEST_FLAGS_USE_DMA; ret = ioctl(fd, PCITEST_COPY, ¶m); - fprintf(stdout, "COPY (%7ld bytes):\t\t", test->size); + fprintf(stdout, "COPY (%7lu bytes):\t\t", test->size); if (ret < 0) fprintf(stdout, "TEST FAILED\n"); else diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore index f5b81d439387..5aaf73df6700 100644 --- a/tools/perf/.gitignore +++ b/tools/perf/.gitignore @@ -39,9 +39,9 @@ trace/beauty/generated/ pmu-events/pmu-events.c pmu-events/jevents pmu-events/metric_test.log -tests/shell/*.shellcheck_log -tests/shell/coresight/*.shellcheck_log -tests/shell/lib/*.shellcheck_log +pmu-events/empty-pmu-events.log +pmu-events/test-empty-pmu-events.c +*.shellcheck_log feature/ libapi/ libbpf/ diff --git a/tools/perf/Documentation/itrace.txt b/tools/perf/Documentation/itrace.txt index 19cc179be9a7..40476b227f8d 100644 --- a/tools/perf/Documentation/itrace.txt +++ b/tools/perf/Documentation/itrace.txt @@ -1,6 +1,6 @@ i synthesize instructions events y synthesize cycles events - b synthesize branches events (branch misses for Arm SPE) + b synthesize branches events c synthesize branches events (calls only) r synthesize branches events (returns only) x synthesize transactions events diff --git a/tools/perf/Documentation/perf-arm-spe.txt b/tools/perf/Documentation/perf-arm-spe.txt index 0a3eda482307..de2b0b479249 100644 --- a/tools/perf/Documentation/perf-arm-spe.txt +++ b/tools/perf/Documentation/perf-arm-spe.txt @@ -187,7 +187,7 @@ groups: 7 llc-access 2 tlb-miss 1K tlb-access - 36 branch-miss + 36 branch 0 remote-access 900 memory diff --git a/tools/perf/Documentation/perf-check.txt b/tools/perf/Documentation/perf-check.txt index 10f69fb6850b..31741499e786 100644 --- a/tools/perf/Documentation/perf-check.txt +++ b/tools/perf/Documentation/perf-check.txt @@ -47,15 +47,15 @@ feature:: bpf / HAVE_LIBBPF_SUPPORT bpf_skeletons / HAVE_BPF_SKEL debuginfod / HAVE_DEBUGINFOD_SUPPORT - dwarf / HAVE_DWARF_SUPPORT - dwarf_getlocations / HAVE_DWARF_GETLOCATIONS_SUPPORT + dwarf / HAVE_LIBDW_SUPPORT + dwarf_getlocations / HAVE_LIBDW_SUPPORT dwarf-unwind / HAVE_DWARF_UNWIND_SUPPORT auxtrace / HAVE_AUXTRACE_SUPPORT libaudit / HAVE_LIBAUDIT_SUPPORT libbfd / HAVE_LIBBFD_SUPPORT libcapstone / HAVE_LIBCAPSTONE_SUPPORT libcrypto / HAVE_LIBCRYPTO_SUPPORT - libdw-dwarf-unwind / HAVE_DWARF_SUPPORT + libdw-dwarf-unwind / HAVE_LIBDW_SUPPORT libelf / HAVE_LIBELF_SUPPORT libnuma / HAVE_LIBNUMA_SUPPORT libopencsd / HAVE_CSTRACE_SUPPORT diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt index 379f9d7a8ab1..1f668d4724e3 100644 --- a/tools/perf/Documentation/perf-config.txt +++ b/tools/perf/Documentation/perf-config.txt @@ -247,6 +247,19 @@ annotate.*:: These are in control of addresses, jump function, source code in lines of assembly code from a specific program. + annotate.disassemblers:: + Choose the disassembler to use: "objdump", "llvm", "capstone", + if not specified it will first try, if available, the "llvm" one, + then, if it fails, "capstone", and finally the original "objdump" + based one. + + Choosing a different one is useful when handling some feature that + is known to be best support at some point by one of the options, + to compare the output when in doubt about some bug, etc. + + This can be a list, in order of preference, the first one that works + finishes the process. + annotate.addr2line:: addr2line binary to use for file names and line numbers. diff --git a/tools/perf/Documentation/perf-list.txt b/tools/perf/Documentation/perf-list.txt index dea005410ec0..d0c65fad419a 100644 --- a/tools/perf/Documentation/perf-list.txt +++ b/tools/perf/Documentation/perf-list.txt @@ -8,7 +8,7 @@ perf-list - List all symbolic event types SYNOPSIS -------- [verse] -'perf list' [--no-desc] [--long-desc] +'perf list' [<options>] [hw|sw|cache|tracepoint|pmu|sdt|metric|metricgroup|event_glob] DESCRIPTION @@ -243,6 +243,21 @@ For accessing trace point events perf needs to have read access to /sys/kernel/tracing, even when perf_event_paranoid is in a relaxed setting. +TOOL/HWMON EVENTS +----------------- + +Some events don't have an associated PMU instead reading values +available to software without perf_event_open. As these events don't +support sampling they can only really be read by tools like perf stat. + +Tool events provide times and certain system parameters. Examples +include duration_time, user_time, system_time and num_cpus_online. + +Hwmon events provide easy access to hwmon sysfs data typically in +/sys/class/hwmon. This information includes temperatures, fan speeds +and energy usage. + + TRACING ------- diff --git a/tools/perf/Documentation/perf-report.txt b/tools/perf/Documentation/perf-report.txt index 7c66d81ab978..87f864519406 100644 --- a/tools/perf/Documentation/perf-report.txt +++ b/tools/perf/Documentation/perf-report.txt @@ -391,6 +391,14 @@ OPTIONS This allows to examine the path the program took to each sample. The data collection must have used -b (or -j) and -g. + Also show with some branch flags that can be: + - Predicted: display the average percentage of predicated branches. + (predicated number / total number) + - Abort: display the number of tsx aborted branches. + - Cycles: cycles in basic block. + + - iterations: display the average number of iterations in callchain list. + --addr2line=<path>:: Path to addr2line binary. diff --git a/tools/perf/Documentation/perf-sched.txt b/tools/perf/Documentation/perf-sched.txt index 3db64954a267..6dbbddb6464d 100644 --- a/tools/perf/Documentation/perf-sched.txt +++ b/tools/perf/Documentation/perf-sched.txt @@ -221,6 +221,14 @@ OPTIONS for 'perf sched timehist' priorities are specified with -: 120-129. A combination of both can also be provided: 0,120-129. +-P:: +--pre-migrations:: + Show pre-migration wait time. pre-migration wait time is the time spent + by a task waiting on a runqueue but not getting the chance to run there + and is migrated to a different runqueue where it is finally run. This + time between sched_wakeup and migrate_task is the pre-migration wait + time. + OPTIONS for 'perf sched replay' ------------------------------ diff --git a/tools/perf/Documentation/perf-script-python.txt b/tools/perf/Documentation/perf-script-python.txt index 13e37e9385ee..27a1cac6fe76 100644 --- a/tools/perf/Documentation/perf-script-python.txt +++ b/tools/perf/Documentation/perf-script-python.txt @@ -624,7 +624,7 @@ as perf_trace_context.perf_script_context . perf_set_itrace_options(context, itrace_options) - set --itrace options if they have not been set already perf_sample_srcline(context) - returns source_file_name, line_number perf_sample_srccode(context) - returns source_file_name, line_number, source_line - + perf_config_get(config_name) - returns the value of the named config item, or None if unset Util.py Module ~~~~~~~~~~~~~~ diff --git a/tools/perf/Documentation/perf-test.txt b/tools/perf/Documentation/perf-test.txt index 9acb8d1f6588..efcdec528a8f 100644 --- a/tools/perf/Documentation/perf-test.txt +++ b/tools/perf/Documentation/perf-test.txt @@ -48,3 +48,20 @@ OPTIONS --dso:: Specify a DSO for the "Symbols" test. + +-w:: +--workload=:: + Run a built-in workload, to list them use '--list-workloads', current ones include: + noploop, thloop, leafloop, sqrtloop, brstack, datasym and landlock. + + Used with the shell script regression tests. + + Some accept an extra parameter: + + seconds: leafloop, noploop, sqrtloop, thloop + nrloops: brstack + + The datasym and landlock workloads don't accept any. + +--list-workloads:: + List the available workloads to use with -w/--workload. diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index d4332675babb..2916d59c88cd 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -31,7 +31,7 @@ $(call detected_var,SRCARCH) ifneq ($(NO_SYSCALL_TABLE),1) NO_SYSCALL_TABLE := 1 - ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 powerpc arm64 s390 mips loongarch)) + ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 powerpc arm64 s390 mips loongarch riscv)) NO_SYSCALL_TABLE := 0 endif @@ -83,6 +83,10 @@ ifeq ($(ARCH),mips) LIBUNWIND_LIBS = -lunwind -lunwind-mips endif +ifeq ($(ARCH),riscv) + CFLAGS += -I$(OUTPUT)arch/riscv/include/generated +endif + # So far there's only x86 and arm libdw unwind support merged in perf. # Disable it on all other architectures in case libdw unwind # support is detected in system. Add supported architectures @@ -91,6 +95,10 @@ ifneq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc s390 csky riscv loon NO_LIBDW_DWARF_UNWIND := 1 endif +ifneq ($(LIBUNWIND),1) + NO_LIBUNWIND := 1 +endif + ifeq ($(LIBUNWIND_LIBS),) NO_LIBUNWIND := 1 endif @@ -162,8 +170,8 @@ ifeq ($(findstring -static,${LDFLAGS}),-static) # Must put -ldl after -lebl for dependency DWARFLIBS += -ldl endif -FEATURE_CHECK_CFLAGS-libdw-dwarf-unwind := $(LIBDW_CFLAGS) -FEATURE_CHECK_LDFLAGS-libdw-dwarf-unwind := $(LIBDW_LDFLAGS) $(DWARFLIBS) +FEATURE_CHECK_CFLAGS-libdw := $(LIBDW_CFLAGS) +FEATURE_CHECK_LDFLAGS-libdw := $(LIBDW_LDFLAGS) $(DWARFLIBS) # for linking with debug library, run like: # make DEBUG=1 LIBBABELTRACE_DIR=/opt/libbabeltrace/ @@ -203,10 +211,6 @@ FEATURE_CHECK_CFLAGS-bpf = -I. -I$(srctree)/tools/include -I$(srctree)/tools/arc # include ARCH specific config -include $(src-perf)/arch/$(SRCARCH)/Makefile -ifdef PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET - CFLAGS += -DHAVE_ARCH_REGS_QUERY_REGISTER_OFFSET -endif - include $(srctree)/tools/scripts/utilities.mak ifeq ($(call get-executable,$(FLEX)),) @@ -426,7 +430,7 @@ ifeq ($(feature-file-handle), 1) endif ifdef NO_LIBELF - NO_DWARF := 1 + NO_LIBDW := 1 NO_LIBUNWIND := 1 NO_LIBDW_DWARF_UNWIND := 1 NO_LIBBPF := 1 @@ -461,28 +465,11 @@ else endif endif else - ifndef NO_LIBDW_DWARF_UNWIND - ifneq ($(feature-libdw-dwarf-unwind),1) - NO_LIBDW_DWARF_UNWIND := 1 - $(warning No libdw DWARF unwind found, Please install elfutils-devel/libdw-dev >= 0.158 and/or set LIBDW_DIR) + ifneq ($(feature-libdw), 1) + ifndef NO_LIBDW + $(warning No libdw.h found or old libdw.h found or elfutils is older than 0.157, disables dwarf support. Please install new elfutils-devel/libdw-dev) + NO_LIBDW := 1 endif - endif - ifneq ($(feature-dwarf), 1) - ifndef NO_DWARF - $(warning No libdw.h found or old libdw.h found or elfutils is older than 0.138, disables dwarf support. Please install new elfutils-devel/libdw-dev) - NO_DWARF := 1 - endif - else - ifneq ($(feature-dwarf_getlocations), 1) - $(warning Old libdw.h, finding variables at given 'perf probe' point will not work, install elfutils-devel/libdw-dev >= 0.157) - else - CFLAGS += -DHAVE_DWARF_GETLOCATIONS_SUPPORT - endif # dwarf_getlocations - ifneq ($(feature-dwarf_getcfi), 1) - $(warning Old libdw.h, finding variables at given 'perf probe' point will not work, install elfutils-devel/libdw-dev >= 0.142) - else - CFLAGS += -DHAVE_DWARF_CFI_SUPPORT - endif # dwarf_getcfi endif # Dwarf support endif # libelf support endif # NO_LIBELF @@ -493,7 +480,7 @@ ifeq ($(feature-libaio), 1) endif endif -ifdef NO_DWARF +ifdef NO_LIBDW NO_LIBDW_DWARF_UNWIND := 1 endif @@ -571,17 +558,12 @@ ifndef NO_LIBELF endif endif - ifndef NO_DWARF - ifeq ($(origin PERF_HAVE_DWARF_REGS), undefined) - $(warning DWARF register mappings have not been defined for architecture $(SRCARCH), DWARF support disabled) - NO_DWARF := 1 - else - CFLAGS += -DHAVE_DWARF_SUPPORT $(LIBDW_CFLAGS) - LDFLAGS += $(LIBDW_LDFLAGS) - EXTLIBS += ${DWARFLIBS} - $(call detected,CONFIG_DWARF) - endif # PERF_HAVE_DWARF_REGS - endif # NO_DWARF + ifndef NO_LIBDW + CFLAGS += -DHAVE_LIBDW_SUPPORT $(LIBDW_CFLAGS) + LDFLAGS += $(LIBDW_LDFLAGS) + EXTLIBS += ${DWARFLIBS} + $(call detected,CONFIG_LIBDW) + endif # NO_LIBDW ifndef NO_LIBBPF ifeq ($(feature-bpf), 1) @@ -630,7 +612,7 @@ ifdef PERF_HAVE_JITDUMP endif ifeq ($(SRCARCH),powerpc) - ifndef NO_DWARF + ifndef NO_LIBDW CFLAGS += -DHAVE_SKIP_CALLCHAIN_IDX endif endif @@ -750,8 +732,6 @@ endif ifeq ($(dwarf-post-unwind),1) CFLAGS += -DHAVE_DWARF_UNWIND_SUPPORT $(call detected,CONFIG_DWARF_UNWIND) -else - NO_DWARF_UNWIND := 1 endif ifndef NO_LOCAL_LIBUNWIND @@ -1194,7 +1174,7 @@ endif ifneq ($(NO_LIBTRACEEVENT),1) $(call feature_check,libtraceevent) ifeq ($(feature-libtraceevent), 1) - CFLAGS += -DHAVE_LIBTRACEEVENT + CFLAGS += -DHAVE_LIBTRACEEVENT $(shell $(PKG_CONFIG) --cflags libtraceevent) LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-L libtraceevent) EXTLIBS += $(shell $(PKG_CONFIG) --libs-only-l libtraceevent) LIBTRACEEVENT_VERSION := $(shell $(PKG_CONFIG) --modversion libtraceevent).0.0 diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 9dd2e8d3f3c9..d74241a15131 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -40,7 +40,7 @@ include ../scripts/utilities.mak # # Define EXTRA_PERFLIBS to pass extra libraries to PERFLIBS. # -# Define NO_DWARF if you do not want debug-info analysis feature at all. +# Define NO_LIBDW if you do not want debug-info analysis feature at all. # # Define WERROR=0 to disable treating any warnings as errors. # @@ -52,7 +52,7 @@ include ../scripts/utilities.mak # # Define NO_LIBELF if you do not want libelf dependency (e.g. cross-builds) # -# Define NO_LIBUNWIND if you do not want libunwind dependency for dwarf +# Define LIBUNWIND if you do not want libunwind dependency for dwarf # backtrace post unwind. # # Define NO_BACKTRACE if you do not want stack backtrace debug feature @@ -1128,12 +1128,11 @@ endif install-tests: all install-gtk $(call QUIET_INSTALL, tests) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ - $(INSTALL) tests/attr.py -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ $(INSTALL) tests/pe-file.exe* '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ - $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/attr'; \ - $(INSTALL) tests/attr/* -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/attr'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \ $(INSTALL) tests/shell/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \ + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/attr'; \ + $(INSTALL) tests/shell/attr/* -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/attr'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ $(INSTALL) tests/shell/lib/*.sh -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ $(INSTALL) tests/shell/lib/*.py -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ diff --git a/tools/perf/arch/arc/annotate/instructions.c b/tools/perf/arch/arc/annotate/instructions.c index 2f00e995c7e3..e5619770a1af 100644 --- a/tools/perf/arch/arc/annotate/instructions.c +++ b/tools/perf/arch/arc/annotate/instructions.c @@ -5,5 +5,7 @@ static int arc__annotate_init(struct arch *arch, char *cpuid __maybe_unused) { arch->initialized = true; arch->objdump.comment_char = ';'; + arch->e_machine = EM_ARC; + arch->e_flags = 0; return 0; } diff --git a/tools/perf/arch/arm/Makefile b/tools/perf/arch/arm/Makefile index 1d88fdab13bf..8b59ce8efb89 100644 --- a/tools/perf/arch/arm/Makefile +++ b/tools/perf/arch/arm/Makefile @@ -1,5 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif PERF_HAVE_JITDUMP := 1 diff --git a/tools/perf/arch/arm/annotate/instructions.c b/tools/perf/arch/arm/annotate/instructions.c index 2ff6cedeb9c5..cf91a43362b0 100644 --- a/tools/perf/arch/arm/annotate/instructions.c +++ b/tools/perf/arch/arm/annotate/instructions.c @@ -53,6 +53,8 @@ static int arm__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->associate_instruction_ops = arm__associate_instruction_ops; arch->objdump.comment_char = ';'; arch->objdump.skip_functions_char = '+'; + arch->e_machine = EM_ARM; + arch->e_flags = 0; return 0; out_free_call: diff --git a/tools/perf/arch/arm/util/Build b/tools/perf/arch/arm/util/Build index e6dd7cd79ebd..f7a8b37d1c68 100644 --- a/tools/perf/arch/arm/util/Build +++ b/tools/perf/arch/arm/util/Build @@ -1,7 +1,5 @@ perf-util-y += perf_regs.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o - perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/arm/util/dwarf-regs.c b/tools/perf/arch/arm/util/dwarf-regs.c deleted file mode 100644 index fc5f71c91802..000000000000 --- a/tools/perf/arch/arm/util/dwarf-regs.c +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2010 Will Deacon, ARM Ltd. - */ - -#include <stddef.h> -#include <linux/stringify.h> -#include <dwarf-regs.h> - -struct pt_regs_dwarfnum { - const char *name; - unsigned int dwarfnum; -}; - -#define REG_DWARFNUM_NAME(r, num) {.name = r, .dwarfnum = num} -#define GPR_DWARFNUM_NAME(num) \ - {.name = __stringify(%r##num), .dwarfnum = num} -#define REG_DWARFNUM_END {.name = NULL, .dwarfnum = 0} - -/* - * Reference: - * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0040a/IHI0040A_aadwarf.pdf - */ -static const struct pt_regs_dwarfnum regdwarfnum_table[] = { - GPR_DWARFNUM_NAME(0), - GPR_DWARFNUM_NAME(1), - GPR_DWARFNUM_NAME(2), - GPR_DWARFNUM_NAME(3), - GPR_DWARFNUM_NAME(4), - GPR_DWARFNUM_NAME(5), - GPR_DWARFNUM_NAME(6), - GPR_DWARFNUM_NAME(7), - GPR_DWARFNUM_NAME(8), - GPR_DWARFNUM_NAME(9), - GPR_DWARFNUM_NAME(10), - REG_DWARFNUM_NAME("%fp", 11), - REG_DWARFNUM_NAME("%ip", 12), - REG_DWARFNUM_NAME("%sp", 13), - REG_DWARFNUM_NAME("%lr", 14), - REG_DWARFNUM_NAME("%pc", 15), - REG_DWARFNUM_END, -}; - -/** - * get_arch_regstr() - lookup register name from it's DWARF register number - * @n: the DWARF register number - * - * get_arch_regstr() returns the name of the register in struct - * regdwarfnum_table from it's DWARF register number. If the register is not - * found in the table, this returns NULL; - */ -const char *get_arch_regstr(unsigned int n) -{ - const struct pt_regs_dwarfnum *roff; - for (roff = regdwarfnum_table; roff->name != NULL; roff++) - if (roff->dwarfnum == n) - return roff->name; - return NULL; -} diff --git a/tools/perf/arch/arm64/Makefile b/tools/perf/arch/arm64/Makefile index 5735ed4479bb..91570d5d428e 100644 --- a/tools/perf/arch/arm64/Makefile +++ b/tools/perf/arch/arm64/Makefile @@ -1,9 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif PERF_HAVE_JITDUMP := 1 -PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 HAVE_KVM_STAT_SUPPORT := 1 # diff --git a/tools/perf/arch/arm64/annotate/instructions.c b/tools/perf/arch/arm64/annotate/instructions.c index f86d9f4798bd..d465d093e7eb 100644 --- a/tools/perf/arch/arm64/annotate/instructions.c +++ b/tools/perf/arch/arm64/annotate/instructions.c @@ -113,6 +113,8 @@ static int arm64__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->associate_instruction_ops = arm64__associate_instruction_ops; arch->objdump.comment_char = '/'; arch->objdump.skip_functions_char = '+'; + arch->e_machine = EM_AARCH64; + arch->e_flags = 0; return 0; out_free_call: diff --git a/tools/perf/arch/arm64/util/Build b/tools/perf/arch/arm64/util/Build index 343ef7589a77..a74521b79eaa 100644 --- a/tools/perf/arch/arm64/util/Build +++ b/tools/perf/arch/arm64/util/Build @@ -4,7 +4,6 @@ perf-util-y += perf_regs.o perf-util-y += tsc.o perf-util-y += pmu.o perf-util-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index 2be99fdf997d..22b19dcc6beb 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -23,9 +23,12 @@ #include "../../../util/debug.h" #include "../../../util/auxtrace.h" #include "../../../util/record.h" +#include "../../../util/header.h" #include "../../../util/arm-spe.h" #include <tools/libc_compat.h> // reallocarray +#define ARM_SPE_CPU_MAGIC 0x1010101010101010ULL + #define KiB(x) ((x) * 1024) #define MiB(x) ((x) * 1024 * 1024) @@ -37,11 +40,84 @@ struct arm_spe_recording { bool *wrapped; }; +/* + * arm_spe_find_cpus() returns a new cpu map, and the caller should invoke + * perf_cpu_map__put() to release the map after use. + */ +static struct perf_cpu_map *arm_spe_find_cpus(struct evlist *evlist) +{ + struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); + struct perf_cpu_map *intersect_cpus; + + /* cpu map is not "any" CPU , we have specific CPUs to work with */ + if (!perf_cpu_map__has_any_cpu(event_cpus)) { + intersect_cpus = perf_cpu_map__intersect(event_cpus, online_cpus); + perf_cpu_map__put(online_cpus); + /* Event can be "any" CPU so count all CPUs. */ + } else { + intersect_cpus = online_cpus; + } + + return intersect_cpus; +} + static size_t arm_spe_info_priv_size(struct auxtrace_record *itr __maybe_unused, - struct evlist *evlist __maybe_unused) + struct evlist *evlist) +{ + struct perf_cpu_map *cpu_map = arm_spe_find_cpus(evlist); + size_t size; + + if (!cpu_map) + return 0; + + size = ARM_SPE_AUXTRACE_PRIV_MAX + + ARM_SPE_CPU_PRIV_MAX * perf_cpu_map__nr(cpu_map); + size *= sizeof(u64); + + perf_cpu_map__put(cpu_map); + return size; +} + +static int arm_spe_save_cpu_header(struct auxtrace_record *itr, + struct perf_cpu cpu, __u64 data[]) { - return ARM_SPE_AUXTRACE_PRIV_SIZE; + struct arm_spe_recording *sper = + container_of(itr, struct arm_spe_recording, itr); + struct perf_pmu *pmu = NULL; + char *cpuid = NULL; + u64 val; + + /* Read CPU MIDR */ + cpuid = get_cpuid_allow_env_override(cpu); + if (!cpuid) + return -ENOMEM; + val = strtol(cpuid, NULL, 16); + + data[ARM_SPE_MAGIC] = ARM_SPE_CPU_MAGIC; + data[ARM_SPE_CPU] = cpu.cpu; + data[ARM_SPE_CPU_NR_PARAMS] = ARM_SPE_CPU_PRIV_MAX - ARM_SPE_CPU_MIDR; + data[ARM_SPE_CPU_MIDR] = val; + + /* Find the associate Arm SPE PMU for the CPU */ + if (perf_cpu_map__has(sper->arm_spe_pmu->cpus, cpu)) + pmu = sper->arm_spe_pmu; + + if (!pmu) { + /* No Arm SPE PMU is found */ + data[ARM_SPE_CPU_PMU_TYPE] = ULLONG_MAX; + data[ARM_SPE_CAP_MIN_IVAL] = 0; + } else { + data[ARM_SPE_CPU_PMU_TYPE] = pmu->type; + + if (perf_pmu__scan_file(pmu, "caps/min_interval", "%lu", &val) != 1) + val = 0; + data[ARM_SPE_CAP_MIN_IVAL] = val; + } + + free(cpuid); + return ARM_SPE_CPU_PRIV_MAX; } static int arm_spe_info_fill(struct auxtrace_record *itr, @@ -49,20 +125,46 @@ static int arm_spe_info_fill(struct auxtrace_record *itr, struct perf_record_auxtrace_info *auxtrace_info, size_t priv_size) { + int i, ret; + size_t offset; struct arm_spe_recording *sper = container_of(itr, struct arm_spe_recording, itr); struct perf_pmu *arm_spe_pmu = sper->arm_spe_pmu; + struct perf_cpu_map *cpu_map; + struct perf_cpu cpu; + __u64 *data; - if (priv_size != ARM_SPE_AUXTRACE_PRIV_SIZE) + if (priv_size != arm_spe_info_priv_size(itr, session->evlist)) return -EINVAL; if (!session->evlist->core.nr_mmaps) return -EINVAL; + cpu_map = arm_spe_find_cpus(session->evlist); + if (!cpu_map) + return -EINVAL; + auxtrace_info->type = PERF_AUXTRACE_ARM_SPE; - auxtrace_info->priv[ARM_SPE_PMU_TYPE] = arm_spe_pmu->type; + auxtrace_info->priv[ARM_SPE_HEADER_VERSION] = ARM_SPE_HEADER_CURRENT_VERSION; + auxtrace_info->priv[ARM_SPE_HEADER_SIZE] = + ARM_SPE_AUXTRACE_PRIV_MAX - ARM_SPE_HEADER_VERSION; + auxtrace_info->priv[ARM_SPE_PMU_TYPE_V2] = arm_spe_pmu->type; + auxtrace_info->priv[ARM_SPE_CPUS_NUM] = perf_cpu_map__nr(cpu_map); + + offset = ARM_SPE_AUXTRACE_PRIV_MAX; + perf_cpu_map__for_each_cpu(cpu, i, cpu_map) { + assert(offset < priv_size); + data = &auxtrace_info->priv[offset]; + ret = arm_spe_save_cpu_header(itr, cpu, data); + if (ret < 0) + goto out; + offset += ret; + } - return 0; + ret = 0; +out: + perf_cpu_map__put(cpu_map); + return ret; } static void @@ -188,9 +290,9 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, evlist__for_each_entry(evlist, evsel) { if (evsel__is_aux_event(evsel)) { - if (!strstarts(evsel->pmu_name, ARM_SPE_PMU_NAME)) { + if (!strstarts(evsel->pmu->name, ARM_SPE_PMU_NAME)) { pr_err("Found unexpected auxtrace event: %s\n", - evsel->pmu_name); + evsel->pmu->name); return -EINVAL; } opts->full_auxtrace = true; diff --git a/tools/perf/arch/arm64/util/dwarf-regs.c b/tools/perf/arch/arm64/util/dwarf-regs.c deleted file mode 100644 index 917b97d7c5d3..000000000000 --- a/tools/perf/arch/arm64/util/dwarf-regs.c +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2010 Will Deacon, ARM Ltd. - */ - -#include <errno.h> -#include <stddef.h> -#include <string.h> -#include <dwarf-regs.h> -#include <linux/ptrace.h> /* for struct user_pt_regs */ -#include <linux/stringify.h> - -struct pt_regs_dwarfnum { - const char *name; - unsigned int dwarfnum; -}; - -#define REG_DWARFNUM_NAME(r, num) {.name = r, .dwarfnum = num} -#define GPR_DWARFNUM_NAME(num) \ - {.name = __stringify(%x##num), .dwarfnum = num} -#define REG_DWARFNUM_END {.name = NULL, .dwarfnum = 0} -#define DWARFNUM2OFFSET(index) \ - (index * sizeof((struct user_pt_regs *)0)->regs[0]) - -/* - * Reference: - * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0057b/IHI0057B_aadwarf64.pdf - */ -static const struct pt_regs_dwarfnum regdwarfnum_table[] = { - GPR_DWARFNUM_NAME(0), - GPR_DWARFNUM_NAME(1), - GPR_DWARFNUM_NAME(2), - GPR_DWARFNUM_NAME(3), - GPR_DWARFNUM_NAME(4), - GPR_DWARFNUM_NAME(5), - GPR_DWARFNUM_NAME(6), - GPR_DWARFNUM_NAME(7), - GPR_DWARFNUM_NAME(8), - GPR_DWARFNUM_NAME(9), - GPR_DWARFNUM_NAME(10), - GPR_DWARFNUM_NAME(11), - GPR_DWARFNUM_NAME(12), - GPR_DWARFNUM_NAME(13), - GPR_DWARFNUM_NAME(14), - GPR_DWARFNUM_NAME(15), - GPR_DWARFNUM_NAME(16), - GPR_DWARFNUM_NAME(17), - GPR_DWARFNUM_NAME(18), - GPR_DWARFNUM_NAME(19), - GPR_DWARFNUM_NAME(20), - GPR_DWARFNUM_NAME(21), - GPR_DWARFNUM_NAME(22), - GPR_DWARFNUM_NAME(23), - GPR_DWARFNUM_NAME(24), - GPR_DWARFNUM_NAME(25), - GPR_DWARFNUM_NAME(26), - GPR_DWARFNUM_NAME(27), - GPR_DWARFNUM_NAME(28), - GPR_DWARFNUM_NAME(29), - REG_DWARFNUM_NAME("%lr", 30), - REG_DWARFNUM_NAME("%sp", 31), - REG_DWARFNUM_END, -}; - -/** - * get_arch_regstr() - lookup register name from it's DWARF register number - * @n: the DWARF register number - * - * get_arch_regstr() returns the name of the register in struct - * regdwarfnum_table from it's DWARF register number. If the register is not - * found in the table, this returns NULL; - */ -const char *get_arch_regstr(unsigned int n) -{ - const struct pt_regs_dwarfnum *roff; - for (roff = regdwarfnum_table; roff->name != NULL; roff++) - if (roff->dwarfnum == n) - return roff->name; - return NULL; -} - -int regs_query_register_offset(const char *name) -{ - const struct pt_regs_dwarfnum *roff; - - for (roff = regdwarfnum_table; roff->name != NULL; roff++) - if (!strcmp(roff->name, name)) - return DWARFNUM2OFFSET(roff->dwarfnum); - return -EINVAL; -} diff --git a/tools/perf/arch/arm64/util/header.c b/tools/perf/arch/arm64/util/header.c index 741df3614a09..f445a2dd6293 100644 --- a/tools/perf/arch/arm64/util/header.c +++ b/tools/perf/arch/arm64/util/header.c @@ -14,73 +14,66 @@ #define MIDR_REVISION_MASK GENMASK(3, 0) #define MIDR_VARIANT_MASK GENMASK(23, 20) -static int _get_cpuid(char *buf, size_t sz, struct perf_cpu_map *cpus) +static int _get_cpuid(char *buf, size_t sz, struct perf_cpu cpu) { + char path[PATH_MAX]; + FILE *file; const char *sysfs = sysfs__mountpoint(); - struct perf_cpu cpu; - int idx, ret = EINVAL; + assert(cpu.cpu != -1); if (!sysfs || sz < MIDR_SIZE) return EINVAL; - perf_cpu_map__for_each_cpu(cpu, idx, cpus) { - char path[PATH_MAX]; - FILE *file; - - scnprintf(path, PATH_MAX, "%s/devices/system/cpu/cpu%d" MIDR, - sysfs, cpu.cpu); - - file = fopen(path, "r"); - if (!file) { - pr_debug("fopen failed for file %s\n", path); - continue; - } - - if (!fgets(buf, MIDR_SIZE, file)) { - fclose(file); - continue; - } - fclose(file); + scnprintf(path, PATH_MAX, "%s/devices/system/cpu/cpu%d" MIDR, sysfs, cpu.cpu); - /* got midr break loop */ - ret = 0; - break; + file = fopen(path, "r"); + if (!file) { + pr_debug("fopen failed for file %s\n", path); + return EINVAL; } - return ret; + if (!fgets(buf, MIDR_SIZE, file)) { + pr_debug("Failed to read file %s\n", path); + fclose(file); + return EINVAL; + } + fclose(file); + return 0; } -int get_cpuid(char *buf, size_t sz) +int get_cpuid(char *buf, size_t sz, struct perf_cpu cpu) { - struct perf_cpu_map *cpus = perf_cpu_map__new_online_cpus(); - int ret; + struct perf_cpu_map *cpus; + int idx; + if (cpu.cpu != -1) + return _get_cpuid(buf, sz, cpu); + + cpus = perf_cpu_map__new_online_cpus(); if (!cpus) return EINVAL; - ret = _get_cpuid(buf, sz, cpus); - - perf_cpu_map__put(cpus); + perf_cpu_map__for_each_cpu(cpu, idx, cpus) { + int ret = _get_cpuid(buf, sz, cpu); - return ret; + if (ret == 0) + return 0; + } + return EINVAL; } -char *get_cpuid_str(struct perf_pmu *pmu) +char *get_cpuid_str(struct perf_cpu cpu) { - char *buf = NULL; + char *buf = malloc(MIDR_SIZE); int res; - if (!pmu || !pmu->cpus) - return NULL; - - buf = malloc(MIDR_SIZE); if (!buf) return NULL; /* read midr from list of cpus mapped to this pmu */ - res = _get_cpuid(buf, MIDR_SIZE, pmu->cpus); + res = get_cpuid(buf, MIDR_SIZE, cpu); if (res) { - pr_err("failed to get cpuid string for PMU %s\n", pmu->name); + pr_err("failed to get cpuid string for CPU %d\n", cpu.cpu); free(buf); buf = NULL; } diff --git a/tools/perf/arch/arm64/util/pmu.c b/tools/perf/arch/arm64/util/pmu.c index 2a4eab2d160e..895fb0d0610c 100644 --- a/tools/perf/arch/arm64/util/pmu.c +++ b/tools/perf/arch/arm64/util/pmu.c @@ -1,30 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 -#include <internal/cpumap.h> -#include "../../../util/cpumap.h" -#include "../../../util/header.h" #include "../../../util/pmu.h" #include "../../../util/pmus.h" +#include "../../../util/tool_pmu.h" #include <api/fs/fs.h> -#include <math.h> -const struct pmu_metrics_table *pmu_metrics_table__find(void) -{ - struct perf_pmu *pmu; - - /* Metrics aren't currently supported on heterogeneous Arm systems */ - if (perf_pmus__num_core_pmus() > 1) - return NULL; - - /* Doesn't matter which one here because they'll all be the same */ - pmu = perf_pmus__find_core_pmu(); - if (pmu) - return perf_pmu__find_metrics_table(pmu); - - return NULL; -} - -double perf_pmu__cpu_slots_per_cycle(void) +u64 tool_pmu__cpu_slots_per_cycle(void) { char path[PATH_MAX]; unsigned long long slots = 0; @@ -41,5 +22,5 @@ double perf_pmu__cpu_slots_per_cycle(void) filename__read_ull(path, &slots); } - return slots ? (double)slots : NAN; + return slots; } diff --git a/tools/perf/arch/csky/Makefile b/tools/perf/arch/csky/Makefile deleted file mode 100644 index 88c08eed9c7b..000000000000 --- a/tools/perf/arch/csky/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif diff --git a/tools/perf/arch/csky/annotate/instructions.c b/tools/perf/arch/csky/annotate/instructions.c index 5337bfb7d5fc..14270311d215 100644 --- a/tools/perf/arch/csky/annotate/instructions.c +++ b/tools/perf/arch/csky/annotate/instructions.c @@ -43,6 +43,11 @@ static int csky__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->initialized = true; arch->objdump.comment_char = '/'; arch->associate_instruction_ops = csky__associate_ins_ops; - + arch->e_machine = EM_CSKY; +#if defined(__CSKYABIV2__) + arch->e_flags = EF_CSKY_ABIV2; +#else + arch->e_flags = EF_CSKY_ABIV1; +#endif return 0; } diff --git a/tools/perf/arch/csky/util/Build b/tools/perf/arch/csky/util/Build index 99d83f41bf43..5e6ea82c4202 100644 --- a/tools/perf/arch/csky/util/Build +++ b/tools/perf/arch/csky/util/Build @@ -1,4 +1,3 @@ perf-util-y += perf_regs.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/loongarch/Makefile b/tools/perf/arch/loongarch/Makefile index c89d6bb6b184..52544d59245b 100644 --- a/tools/perf/arch/loongarch/Makefile +++ b/tools/perf/arch/loongarch/Makefile @@ -1,8 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif -PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 PERF_HAVE_JITDUMP := 1 HAVE_KVM_STAT_SUPPORT := 1 diff --git a/tools/perf/arch/loongarch/annotate/instructions.c b/tools/perf/arch/loongarch/annotate/instructions.c index ab43b1ab51e3..70262d5f1444 100644 --- a/tools/perf/arch/loongarch/annotate/instructions.c +++ b/tools/perf/arch/loongarch/annotate/instructions.c @@ -131,6 +131,8 @@ int loongarch__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->associate_instruction_ops = loongarch__associate_ins_ops; arch->initialized = true; arch->objdump.comment_char = '#'; + arch->e_machine = EM_LOONGARCH; + arch->e_flags = 0; } return 0; diff --git a/tools/perf/arch/loongarch/util/Build b/tools/perf/arch/loongarch/util/Build index b6b97de48233..0aa31986ecb5 100644 --- a/tools/perf/arch/loongarch/util/Build +++ b/tools/perf/arch/loongarch/util/Build @@ -1,7 +1,6 @@ perf-util-y += header.o perf-util-y += perf_regs.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o perf-util-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o diff --git a/tools/perf/arch/loongarch/util/dwarf-regs.c b/tools/perf/arch/loongarch/util/dwarf-regs.c deleted file mode 100644 index 0f6ebc387463..000000000000 --- a/tools/perf/arch/loongarch/util/dwarf-regs.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2020-2023 Loongson Technology Corporation Limited - */ - -#include <stdio.h> -#include <errno.h> /* for EINVAL */ -#include <string.h> /* for strcmp */ -#include <dwarf-regs.h> - -struct pt_regs_dwarfnum { - const char *name; - unsigned int dwarfnum; -}; - -static struct pt_regs_dwarfnum loongarch_gpr_table[] = { - {"%r0", 0}, {"%r1", 1}, {"%r2", 2}, {"%r3", 3}, - {"%r4", 4}, {"%r5", 5}, {"%r6", 6}, {"%r7", 7}, - {"%r8", 8}, {"%r9", 9}, {"%r10", 10}, {"%r11", 11}, - {"%r12", 12}, {"%r13", 13}, {"%r14", 14}, {"%r15", 15}, - {"%r16", 16}, {"%r17", 17}, {"%r18", 18}, {"%r19", 19}, - {"%r20", 20}, {"%r21", 21}, {"%r22", 22}, {"%r23", 23}, - {"%r24", 24}, {"%r25", 25}, {"%r26", 26}, {"%r27", 27}, - {"%r28", 28}, {"%r29", 29}, {"%r30", 30}, {"%r31", 31}, - {NULL, 0} -}; - -const char *get_arch_regstr(unsigned int n) -{ - n %= 32; - return loongarch_gpr_table[n].name; -} - -int regs_query_register_offset(const char *name) -{ - const struct pt_regs_dwarfnum *roff; - - for (roff = loongarch_gpr_table; roff->name != NULL; roff++) - if (!strcmp(roff->name, name)) - return roff->dwarfnum; - return -EINVAL; -} diff --git a/tools/perf/arch/loongarch/util/header.c b/tools/perf/arch/loongarch/util/header.c index d962dff55512..0c6d823334a2 100644 --- a/tools/perf/arch/loongarch/util/header.c +++ b/tools/perf/arch/loongarch/util/header.c @@ -70,7 +70,7 @@ out_free: return cpuid; } -int get_cpuid(char *buffer, size_t sz) +int get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu __maybe_unused) { int ret = 0; char *cpuid = _get_cpuid(); @@ -90,7 +90,7 @@ out_free: return ret; } -char *get_cpuid_str(struct perf_pmu *pmu __maybe_unused) +char *get_cpuid_str(struct perf_cpu cpu __maybe_unused) { return _get_cpuid(); } diff --git a/tools/perf/arch/mips/Makefile b/tools/perf/arch/mips/Makefile index cd0b011b3be5..827168f1077a 100644 --- a/tools/perf/arch/mips/Makefile +++ b/tools/perf/arch/mips/Makefile @@ -1,8 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif - # Syscall table generation for perf out := $(OUTPUT)arch/mips/include/generated/asm header := $(out)/syscalls_n64.c diff --git a/tools/perf/arch/mips/annotate/instructions.c b/tools/perf/arch/mips/annotate/instructions.c index 340993f2a897..b50b46c613d6 100644 --- a/tools/perf/arch/mips/annotate/instructions.c +++ b/tools/perf/arch/mips/annotate/instructions.c @@ -40,6 +40,8 @@ int mips__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->associate_instruction_ops = mips__associate_ins_ops; arch->initialized = true; arch->objdump.comment_char = '#'; + arch->e_machine = EM_MIPS; + arch->e_flags = 0; } return 0; diff --git a/tools/perf/arch/mips/util/Build b/tools/perf/arch/mips/util/Build index e4644f1e68a0..691fa2051958 100644 --- a/tools/perf/arch/mips/util/Build +++ b/tools/perf/arch/mips/util/Build @@ -1,3 +1,2 @@ perf-util-y += perf_regs.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o diff --git a/tools/perf/arch/mips/util/dwarf-regs.c b/tools/perf/arch/mips/util/dwarf-regs.c deleted file mode 100644 index 25c13a91c2a7..000000000000 --- a/tools/perf/arch/mips/util/dwarf-regs.c +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2013 Cavium, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#include <stdio.h> -#include <dwarf-regs.h> - -static const char *mips_gpr_names[32] = { - "$0", "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", "$9", - "$10", "$11", "$12", "$13", "$14", "$15", "$16", "$17", "$18", "$19", - "$20", "$21", "$22", "$23", "$24", "$25", "$26", "$27", "$28", "$29", - "$30", "$31" -}; - -const char *get_arch_regstr(unsigned int n) -{ - if (n < 32) - return mips_gpr_names[n]; - if (n == 64) - return "hi"; - if (n == 65) - return "lo"; - return NULL; -} diff --git a/tools/perf/arch/powerpc/Makefile b/tools/perf/arch/powerpc/Makefile index bf6d323574f6..dc8f4fb8e324 100644 --- a/tools/perf/arch/powerpc/Makefile +++ b/tools/perf/arch/powerpc/Makefile @@ -1,10 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif - HAVE_KVM_STAT_SUPPORT := 1 -PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 PERF_HAVE_JITDUMP := 1 # diff --git a/tools/perf/arch/powerpc/annotate/instructions.c b/tools/perf/arch/powerpc/annotate/instructions.c index ede9eeade0ab..ca567cfdcbdb 100644 --- a/tools/perf/arch/powerpc/annotate/instructions.c +++ b/tools/perf/arch/powerpc/annotate/instructions.c @@ -255,7 +255,7 @@ static struct ins_ops *check_ppc_insn(struct disasm_line *dl) * is moved to r31. update_insn_state_powerpc tracks these state * changes */ -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT static void update_insn_state_powerpc(struct type_state *state, struct data_loc_info *dloc, Dwarf_Die * cu_die __maybe_unused, struct disasm_line *dl) @@ -300,7 +300,7 @@ static void update_insn_state_powerpc(struct type_state *state, insn_offset, src->reg1, dst->reg1); pr_debug_type_name(&tsr->type, tsr->kind); } -#endif /* HAVE_DWARF_SUPPORT */ +#endif /* HAVE_LIBDW_SUPPORT */ static int powerpc__annotate_init(struct arch *arch, char *cpuid __maybe_unused) { @@ -309,6 +309,8 @@ static int powerpc__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->associate_instruction_ops = powerpc__associate_instruction_ops; arch->objdump.comment_char = '#'; annotate_opts.show_asm_raw = true; + arch->e_machine = EM_PPC; + arch->e_flags = 0; } return 0; diff --git a/tools/perf/arch/powerpc/util/Build b/tools/perf/arch/powerpc/util/Build index 6c588ecdf3bd..ed82715080f9 100644 --- a/tools/perf/arch/powerpc/util/Build +++ b/tools/perf/arch/powerpc/util/Build @@ -7,8 +7,7 @@ perf-util-y += sym-handling.o perf-util-y += evsel.o perf-util-y += event.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o -perf-util-$(CONFIG_DWARF) += skip-callchain-idx.o +perf-util-$(CONFIG_LIBDW) += skip-callchain-idx.o perf-util-$(CONFIG_LIBUNWIND) += unwind-libunwind.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/powerpc/util/dwarf-regs.c b/tools/perf/arch/powerpc/util/dwarf-regs.c deleted file mode 100644 index 104c7ae5c433..000000000000 --- a/tools/perf/arch/powerpc/util/dwarf-regs.c +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2010 Ian Munsie, IBM Corporation. - */ - -#include <stddef.h> -#include <errno.h> -#include <string.h> -#include <dwarf-regs.h> -#include <linux/ptrace.h> -#include <linux/kernel.h> -#include <linux/stringify.h> - -struct pt_regs_dwarfnum { - const char *name; - unsigned int dwarfnum; - unsigned int ptregs_offset; -}; - -#define REG_DWARFNUM_NAME(r, num) \ - {.name = __stringify(%)__stringify(r), .dwarfnum = num, \ - .ptregs_offset = offsetof(struct pt_regs, r)} -#define GPR_DWARFNUM_NAME(num) \ - {.name = __stringify(%gpr##num), .dwarfnum = num, \ - .ptregs_offset = offsetof(struct pt_regs, gpr[num])} -#define REG_DWARFNUM_END {.name = NULL, .dwarfnum = 0, .ptregs_offset = 0} - -/* - * Reference: - * http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi-1.9.html - */ -static const struct pt_regs_dwarfnum regdwarfnum_table[] = { - GPR_DWARFNUM_NAME(0), - GPR_DWARFNUM_NAME(1), - GPR_DWARFNUM_NAME(2), - GPR_DWARFNUM_NAME(3), - GPR_DWARFNUM_NAME(4), - GPR_DWARFNUM_NAME(5), - GPR_DWARFNUM_NAME(6), - GPR_DWARFNUM_NAME(7), - GPR_DWARFNUM_NAME(8), - GPR_DWARFNUM_NAME(9), - GPR_DWARFNUM_NAME(10), - GPR_DWARFNUM_NAME(11), - GPR_DWARFNUM_NAME(12), - GPR_DWARFNUM_NAME(13), - GPR_DWARFNUM_NAME(14), - GPR_DWARFNUM_NAME(15), - GPR_DWARFNUM_NAME(16), - GPR_DWARFNUM_NAME(17), - GPR_DWARFNUM_NAME(18), - GPR_DWARFNUM_NAME(19), - GPR_DWARFNUM_NAME(20), - GPR_DWARFNUM_NAME(21), - GPR_DWARFNUM_NAME(22), - GPR_DWARFNUM_NAME(23), - GPR_DWARFNUM_NAME(24), - GPR_DWARFNUM_NAME(25), - GPR_DWARFNUM_NAME(26), - GPR_DWARFNUM_NAME(27), - GPR_DWARFNUM_NAME(28), - GPR_DWARFNUM_NAME(29), - GPR_DWARFNUM_NAME(30), - GPR_DWARFNUM_NAME(31), - REG_DWARFNUM_NAME(msr, 66), - REG_DWARFNUM_NAME(ctr, 109), - REG_DWARFNUM_NAME(link, 108), - REG_DWARFNUM_NAME(xer, 101), - REG_DWARFNUM_NAME(dar, 119), - REG_DWARFNUM_NAME(dsisr, 118), - REG_DWARFNUM_END, -}; - -/** - * get_arch_regstr() - lookup register name from it's DWARF register number - * @n: the DWARF register number - * - * get_arch_regstr() returns the name of the register in struct - * regdwarfnum_table from it's DWARF register number. If the register is not - * found in the table, this returns NULL; - */ -const char *get_arch_regstr(unsigned int n) -{ - const struct pt_regs_dwarfnum *roff; - for (roff = regdwarfnum_table; roff->name != NULL; roff++) - if (roff->dwarfnum == n) - return roff->name; - return NULL; -} - -int regs_query_register_offset(const char *name) -{ - const struct pt_regs_dwarfnum *roff; - for (roff = regdwarfnum_table; roff->name != NULL; roff++) - if (!strcmp(roff->name, name)) - return roff->ptregs_offset; - return -EINVAL; -} - -#define PPC_OP(op) (((op) >> 26) & 0x3F) -#define PPC_RA(a) (((a) >> 16) & 0x1f) -#define PPC_RT(t) (((t) >> 21) & 0x1f) -#define PPC_RB(b) (((b) >> 11) & 0x1f) -#define PPC_D(D) ((D) & 0xfffe) -#define PPC_DS(DS) ((DS) & 0xfffc) -#define OP_LD 58 -#define OP_STD 62 - -static int get_source_reg(u32 raw_insn) -{ - return PPC_RA(raw_insn); -} - -static int get_target_reg(u32 raw_insn) -{ - return PPC_RT(raw_insn); -} - -static int get_offset_opcode(u32 raw_insn) -{ - int opcode = PPC_OP(raw_insn); - - /* DS- form */ - if ((opcode == OP_LD) || (opcode == OP_STD)) - return PPC_DS(raw_insn); - else - return PPC_D(raw_insn); -} - -/* - * Fills the required fields for op_loc depending on if it - * is a source or target. - * D form: ins RT,D(RA) -> src_reg1 = RA, offset = D, dst_reg1 = RT - * DS form: ins RT,DS(RA) -> src_reg1 = RA, offset = DS, dst_reg1 = RT - * X form: ins RT,RA,RB -> src_reg1 = RA, src_reg2 = RB, dst_reg1 = RT - */ -void get_powerpc_regs(u32 raw_insn, int is_source, - struct annotated_op_loc *op_loc) -{ - if (is_source) - op_loc->reg1 = get_source_reg(raw_insn); - else - op_loc->reg1 = get_target_reg(raw_insn); - - if (op_loc->multi_regs) - op_loc->reg2 = PPC_RB(raw_insn); - - /* TODO: Implement offset handling for X Form */ - if ((op_loc->mem_ref) && (PPC_OP(raw_insn) != 31)) - op_loc->offset = get_offset_opcode(raw_insn); -} diff --git a/tools/perf/arch/powerpc/util/header.c b/tools/perf/arch/powerpc/util/header.c index 6b00efd53638..c7df534dbf8f 100644 --- a/tools/perf/arch/powerpc/util/header.c +++ b/tools/perf/arch/powerpc/util/header.c @@ -10,9 +10,21 @@ #include "utils_header.h" #include "metricgroup.h" #include <api/fs/fs.h> +#include <sys/auxv.h> + +static bool is_compat_mode(void) +{ + u64 base_platform = getauxval(AT_BASE_PLATFORM); + u64 platform = getauxval(AT_PLATFORM); + + if (!strcmp((char *)platform, (char *)base_platform)) + return false; + + return true; +} int -get_cpuid(char *buffer, size_t sz) +get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu __maybe_unused) { unsigned long pvr; int nb; @@ -30,11 +42,29 @@ get_cpuid(char *buffer, size_t sz) } char * -get_cpuid_str(struct perf_pmu *pmu __maybe_unused) +get_cpuid_str(struct perf_cpu cpu __maybe_unused) { char *bufp; + unsigned long pvr; + + /* + * IBM Power System supports compatible mode. That is + * Nth generation platform can support previous generation + * OS in a mode called compatibile mode. For ex. LPAR can be + * booted in a Power9 mode when the system is a Power10. + * + * In the compatible mode, care must be taken when generating + * PVR value. When read, PVR will be of the AT_BASE_PLATFORM + * To support generic events, return 0x00ffffff as pvr when + * booted in compat mode. Based on this pvr value, json will + * pick events from pmu-events/arch/powerpc/compat + */ + if (!is_compat_mode()) + pvr = mfspr(SPRN_PVR); + else + pvr = 0x00ffffff; - if (asprintf(&bufp, "0x%.8lx", mfspr(SPRN_PVR)) < 0) + if (asprintf(&bufp, "0x%.8lx", pvr) < 0) bufp = NULL; return bufp; diff --git a/tools/perf/arch/riscv/Makefile b/tools/perf/arch/riscv/Makefile index 90c3c476a242..18ad078000e2 100644 --- a/tools/perf/arch/riscv/Makefile +++ b/tools/perf/arch/riscv/Makefile @@ -1,6 +1,25 @@ -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif -PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 +# SPDX-License-Identifier: GPL-2.0 PERF_HAVE_JITDUMP := 1 HAVE_KVM_STAT_SUPPORT := 1 + +# +# Syscall table generation for perf +# + +out := $(OUTPUT)arch/riscv/include/generated/asm +header := $(out)/syscalls.c +incpath := $(srctree)/tools +sysdef := $(srctree)/tools/arch/riscv/include/uapi/asm/unistd.h +sysprf := $(srctree)/tools/perf/arch/riscv/entry/syscalls/ +systbl := $(sysprf)/mksyscalltbl + +# Create output directory if not already present +$(shell [ -d '$(out)' ] || mkdir -p '$(out)') + +$(header): $(sysdef) $(systbl) + $(Q)$(SHELL) '$(systbl)' '$(CC)' '$(HOSTCC)' $(incpath) $(sysdef) > $@ + +clean:: + $(call QUIET_CLEAN, riscv) $(RM) $(header) + +archheaders: $(header) diff --git a/tools/perf/arch/riscv/entry/syscalls/mksyscalltbl b/tools/perf/arch/riscv/entry/syscalls/mksyscalltbl new file mode 100755 index 000000000000..c59f5e852b97 --- /dev/null +++ b/tools/perf/arch/riscv/entry/syscalls/mksyscalltbl @@ -0,0 +1,47 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Generate system call table for perf. Derived from +# powerpc script. +# +# Copyright IBM Corp. 2017 +# Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> +# Changed by: Ravi Bangoria <ravi.bangoria@linux.vnet.ibm.com> +# Changed by: Kim Phillips <kim.phillips@arm.com> +# Changed by: Björn Töpel <bjorn@rivosinc.com> + +gcc=$1 +hostcc=$2 +incpath=$3 +input=$4 + +if ! test -r $input; then + echo "Could not read input file" >&2 + exit 1 +fi + +create_sc_table() +{ + local sc nr max_nr + + while read sc nr; do + printf "%s\n" " [$nr] = \"$sc\"," + max_nr=$nr + done + + echo "#define SYSCALLTBL_RISCV_MAX_ID $max_nr" +} + +create_table() +{ + echo "#include \"$input\"" + echo "static const char *const syscalltbl_riscv[] = {" + create_sc_table + echo "};" +} + +$gcc -E -dM -x c -I $incpath/include/uapi $input \ + |awk '$2 ~ "__NR" && $3 !~ "__NR3264_" { + sub("^#define __NR(3264)?_", ""); + print | "sort -k2 -n"}' \ + |create_table diff --git a/tools/perf/arch/riscv/include/dwarf-regs-table.h b/tools/perf/arch/riscv/include/dwarf-regs-table.h new file mode 100644 index 000000000000..a45b63a6d5a8 --- /dev/null +++ b/tools/perf/arch/riscv/include/dwarf-regs-table.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef DEFINE_DWARF_REGSTR_TABLE +/* This is included in perf/util/dwarf-regs.c */ + +#define REG_DWARFNUM_NAME(reg, idx) [idx] = "%" #reg + +static const char * const riscv_regstr_tbl[] = { + REG_DWARFNUM_NAME("%zero", 0), + REG_DWARFNUM_NAME("%ra", 1), + REG_DWARFNUM_NAME("%sp", 2), + REG_DWARFNUM_NAME("%gp", 3), + REG_DWARFNUM_NAME("%tp", 4), + REG_DWARFNUM_NAME("%t0", 5), + REG_DWARFNUM_NAME("%t1", 6), + REG_DWARFNUM_NAME("%t2", 7), + REG_DWARFNUM_NAME("%s0", 8), + REG_DWARFNUM_NAME("%s1", 9), + REG_DWARFNUM_NAME("%a0", 10), + REG_DWARFNUM_NAME("%a1", 11), + REG_DWARFNUM_NAME("%a2", 12), + REG_DWARFNUM_NAME("%a3", 13), + REG_DWARFNUM_NAME("%a4", 14), + REG_DWARFNUM_NAME("%a5", 15), + REG_DWARFNUM_NAME("%a6", 16), + REG_DWARFNUM_NAME("%a7", 17), + REG_DWARFNUM_NAME("%s2", 18), + REG_DWARFNUM_NAME("%s3", 19), + REG_DWARFNUM_NAME("%s4", 20), + REG_DWARFNUM_NAME("%s5", 21), + REG_DWARFNUM_NAME("%s6", 22), + REG_DWARFNUM_NAME("%s7", 23), + REG_DWARFNUM_NAME("%s8", 24), + REG_DWARFNUM_NAME("%s9", 25), + REG_DWARFNUM_NAME("%s10", 26), + REG_DWARFNUM_NAME("%s11", 27), + REG_DWARFNUM_NAME("%t3", 28), + REG_DWARFNUM_NAME("%t4", 29), + REG_DWARFNUM_NAME("%t5", 30), + REG_DWARFNUM_NAME("%t6", 31), +}; + +#endif diff --git a/tools/perf/arch/riscv/util/Build b/tools/perf/arch/riscv/util/Build index f865cb0489ec..58a672246024 100644 --- a/tools/perf/arch/riscv/util/Build +++ b/tools/perf/arch/riscv/util/Build @@ -2,5 +2,4 @@ perf-util-y += perf_regs.o perf-util-y += header.o perf-util-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/riscv/util/dwarf-regs.c b/tools/perf/arch/riscv/util/dwarf-regs.c deleted file mode 100644 index cd0504c02e2e..000000000000 --- a/tools/perf/arch/riscv/util/dwarf-regs.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. - * Mapping of DWARF debug register numbers into register names. - */ - -#include <stddef.h> -#include <errno.h> /* for EINVAL */ -#include <string.h> /* for strcmp */ -#include <dwarf-regs.h> - -struct pt_regs_dwarfnum { - const char *name; - unsigned int dwarfnum; -}; - -#define REG_DWARFNUM_NAME(r, num) {.name = r, .dwarfnum = num} -#define REG_DWARFNUM_END {.name = NULL, .dwarfnum = 0} - -struct pt_regs_dwarfnum riscv_dwarf_regs_table[] = { - REG_DWARFNUM_NAME("%zero", 0), - REG_DWARFNUM_NAME("%ra", 1), - REG_DWARFNUM_NAME("%sp", 2), - REG_DWARFNUM_NAME("%gp", 3), - REG_DWARFNUM_NAME("%tp", 4), - REG_DWARFNUM_NAME("%t0", 5), - REG_DWARFNUM_NAME("%t1", 6), - REG_DWARFNUM_NAME("%t2", 7), - REG_DWARFNUM_NAME("%s0", 8), - REG_DWARFNUM_NAME("%s1", 9), - REG_DWARFNUM_NAME("%a0", 10), - REG_DWARFNUM_NAME("%a1", 11), - REG_DWARFNUM_NAME("%a2", 12), - REG_DWARFNUM_NAME("%a3", 13), - REG_DWARFNUM_NAME("%a4", 14), - REG_DWARFNUM_NAME("%a5", 15), - REG_DWARFNUM_NAME("%a6", 16), - REG_DWARFNUM_NAME("%a7", 17), - REG_DWARFNUM_NAME("%s2", 18), - REG_DWARFNUM_NAME("%s3", 19), - REG_DWARFNUM_NAME("%s4", 20), - REG_DWARFNUM_NAME("%s5", 21), - REG_DWARFNUM_NAME("%s6", 22), - REG_DWARFNUM_NAME("%s7", 23), - REG_DWARFNUM_NAME("%s8", 24), - REG_DWARFNUM_NAME("%s9", 25), - REG_DWARFNUM_NAME("%s10", 26), - REG_DWARFNUM_NAME("%s11", 27), - REG_DWARFNUM_NAME("%t3", 28), - REG_DWARFNUM_NAME("%t4", 29), - REG_DWARFNUM_NAME("%t5", 30), - REG_DWARFNUM_NAME("%t6", 31), - REG_DWARFNUM_END, -}; - -#define RISCV_MAX_REGS ((sizeof(riscv_dwarf_regs_table) / \ - sizeof(riscv_dwarf_regs_table[0])) - 1) - -const char *get_arch_regstr(unsigned int n) -{ - return (n < RISCV_MAX_REGS) ? riscv_dwarf_regs_table[n].name : NULL; -} - -int regs_query_register_offset(const char *name) -{ - const struct pt_regs_dwarfnum *roff; - - for (roff = riscv_dwarf_regs_table; roff->name; roff++) - if (!strcmp(roff->name, name)) - return roff->dwarfnum; - return -EINVAL; -} diff --git a/tools/perf/arch/riscv/util/header.c b/tools/perf/arch/riscv/util/header.c index 1b29030021ee..4b839203d4a5 100644 --- a/tools/perf/arch/riscv/util/header.c +++ b/tools/perf/arch/riscv/util/header.c @@ -81,7 +81,7 @@ free: return cpuid; } -int get_cpuid(char *buffer, size_t sz) +int get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu __maybe_unused) { char *cpuid = _get_cpuid(); int ret = 0; @@ -98,7 +98,7 @@ free: } char * -get_cpuid_str(struct perf_pmu *pmu __maybe_unused) +get_cpuid_str(struct perf_cpu cpu __maybe_unused) { return _get_cpuid(); } diff --git a/tools/perf/arch/riscv64/annotate/instructions.c b/tools/perf/arch/riscv64/annotate/instructions.c index 869a0eb28953..55cf911633f8 100644 --- a/tools/perf/arch/riscv64/annotate/instructions.c +++ b/tools/perf/arch/riscv64/annotate/instructions.c @@ -28,6 +28,8 @@ int riscv64__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->associate_instruction_ops = riscv64__associate_ins_ops; arch->initialized = true; arch->objdump.comment_char = '#'; + arch->e_machine = EM_RISCV; + arch->e_flags = 0; } return 0; diff --git a/tools/perf/arch/s390/Makefile b/tools/perf/arch/s390/Makefile index 56994e63b43a..c431c21b11ef 100644 --- a/tools/perf/arch/s390/Makefile +++ b/tools/perf/arch/s390/Makefile @@ -1,9 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif HAVE_KVM_STAT_SUPPORT := 1 -PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 PERF_HAVE_JITDUMP := 1 # diff --git a/tools/perf/arch/s390/annotate/instructions.c b/tools/perf/arch/s390/annotate/instructions.c index eeac25cca699..c61193f1e096 100644 --- a/tools/perf/arch/s390/annotate/instructions.c +++ b/tools/perf/arch/s390/annotate/instructions.c @@ -166,6 +166,8 @@ static int s390__annotate_init(struct arch *arch, char *cpuid __maybe_unused) if (s390__cpuid_parse(arch, cpuid)) err = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING; } + arch->e_machine = EM_S390; + arch->e_flags = 0; } return err; diff --git a/tools/perf/arch/s390/util/Build b/tools/perf/arch/s390/util/Build index 1ac830030ff3..736c0ad09194 100644 --- a/tools/perf/arch/s390/util/Build +++ b/tools/perf/arch/s390/util/Build @@ -2,7 +2,6 @@ perf-util-y += header.o perf-util-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o perf-util-y += perf_regs.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o perf-util-y += machine.o diff --git a/tools/perf/arch/s390/util/dwarf-regs.c b/tools/perf/arch/s390/util/dwarf-regs.c deleted file mode 100644 index dfddb3099bfa..000000000000 --- a/tools/perf/arch/s390/util/dwarf-regs.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright IBM Corp. 2010, 2017 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> - * - */ - -#include <errno.h> -#include <stddef.h> -#include <stdlib.h> -#include <linux/kernel.h> -#include <asm/ptrace.h> -#include <string.h> -#include <dwarf-regs.h> -#include "dwarf-regs-table.h" - -const char *get_arch_regstr(unsigned int n) -{ - return (n >= ARRAY_SIZE(s390_dwarf_regs)) ? NULL : s390_dwarf_regs[n]; -} - -/* - * Convert the register name into an offset to struct pt_regs (kernel). - * This is required by the BPF prologue generator. The BPF - * program is called in the BPF overflow handler in the perf - * core. - */ -int regs_query_register_offset(const char *name) -{ - unsigned long gpr; - - if (!name || strncmp(name, "%r", 2)) - return -EINVAL; - - errno = 0; - gpr = strtoul(name + 2, NULL, 10); - if (errno || gpr >= 16) - return -EINVAL; - - return offsetof(user_pt_regs, gprs) + 8 * gpr; -} diff --git a/tools/perf/arch/s390/util/header.c b/tools/perf/arch/s390/util/header.c index 7933f6871c81..db54677a17d2 100644 --- a/tools/perf/arch/s390/util/header.c +++ b/tools/perf/arch/s390/util/header.c @@ -27,7 +27,7 @@ #define SYSINFO "/proc/sysinfo" #define SRVLVL "/proc/service_levels" -int get_cpuid(char *buffer, size_t sz) +int get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu __maybe_unused) { char *cp, *line = NULL, *line2; char type[8], model[33], version[8], manufacturer[32], authorization[8]; @@ -137,11 +137,11 @@ skip_sysinfo: return (nbytes >= sz) ? ENOBUFS : 0; } -char *get_cpuid_str(struct perf_pmu *pmu __maybe_unused) +char *get_cpuid_str(struct perf_cpu cpu) { char *buf = malloc(128); - if (buf && get_cpuid(buf, 128)) + if (buf && get_cpuid(buf, 128, cpu)) zfree(&buf); return buf; } diff --git a/tools/perf/arch/sh/Build b/tools/perf/arch/sh/Build deleted file mode 100644 index e63eabc2c8f4..000000000000 --- a/tools/perf/arch/sh/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-y += util/ diff --git a/tools/perf/arch/sh/Makefile b/tools/perf/arch/sh/Makefile deleted file mode 100644 index 88c08eed9c7b..000000000000 --- a/tools/perf/arch/sh/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif diff --git a/tools/perf/arch/sh/util/Build b/tools/perf/arch/sh/util/Build deleted file mode 100644 index 32f44fc4ab98..000000000000 --- a/tools/perf/arch/sh/util/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-$(CONFIG_DWARF) += dwarf-regs.o diff --git a/tools/perf/arch/sh/util/dwarf-regs.c b/tools/perf/arch/sh/util/dwarf-regs.c deleted file mode 100644 index 4b17fc86c73b..000000000000 --- a/tools/perf/arch/sh/util/dwarf-regs.c +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2010 Matt Fleming <matt@console-pimps.org> - */ - -#include <stddef.h> -#include <dwarf-regs.h> - -/* - * Generic dwarf analysis helpers - */ - -#define SH_MAX_REGS 18 -const char *sh_regs_table[SH_MAX_REGS] = { - "r0", - "r1", - "r2", - "r3", - "r4", - "r5", - "r6", - "r7", - "r8", - "r9", - "r10", - "r11", - "r12", - "r13", - "r14", - "r15", - "pc", - "pr", -}; - -/* Return architecture dependent register string (for kprobe-tracer) */ -const char *get_arch_regstr(unsigned int n) -{ - return (n < SH_MAX_REGS) ? sh_regs_table[n] : NULL; -} diff --git a/tools/perf/arch/sparc/Build b/tools/perf/arch/sparc/Build deleted file mode 100644 index e63eabc2c8f4..000000000000 --- a/tools/perf/arch/sparc/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-y += util/ diff --git a/tools/perf/arch/sparc/Makefile b/tools/perf/arch/sparc/Makefile index 4031db72ba71..8b59ce8efb89 100644 --- a/tools/perf/arch/sparc/Makefile +++ b/tools/perf/arch/sparc/Makefile @@ -1,6 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif - PERF_HAVE_JITDUMP := 1 diff --git a/tools/perf/arch/sparc/annotate/instructions.c b/tools/perf/arch/sparc/annotate/instructions.c index 2614c010c235..68c31580ccfc 100644 --- a/tools/perf/arch/sparc/annotate/instructions.c +++ b/tools/perf/arch/sparc/annotate/instructions.c @@ -163,6 +163,8 @@ static int sparc__annotate_init(struct arch *arch, char *cpuid __maybe_unused) arch->initialized = true; arch->associate_instruction_ops = sparc__associate_instruction_ops; arch->objdump.comment_char = '#'; + arch->e_machine = EM_SPARC; + arch->e_flags = 0; } return 0; diff --git a/tools/perf/arch/sparc/util/Build b/tools/perf/arch/sparc/util/Build deleted file mode 100644 index 32f44fc4ab98..000000000000 --- a/tools/perf/arch/sparc/util/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-$(CONFIG_DWARF) += dwarf-regs.o diff --git a/tools/perf/arch/sparc/util/dwarf-regs.c b/tools/perf/arch/sparc/util/dwarf-regs.c deleted file mode 100644 index 1282cb2dc7bd..000000000000 --- a/tools/perf/arch/sparc/util/dwarf-regs.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright (C) 2010 David S. Miller <davem@davemloft.net> - */ - -#include <stddef.h> -#include <dwarf-regs.h> - -#define SPARC_MAX_REGS 96 - -const char *sparc_regs_table[SPARC_MAX_REGS] = { - "%g0", "%g1", "%g2", "%g3", "%g4", "%g5", "%g6", "%g7", - "%o0", "%o1", "%o2", "%o3", "%o4", "%o5", "%sp", "%o7", - "%l0", "%l1", "%l2", "%l3", "%l4", "%l5", "%l6", "%l7", - "%i0", "%i1", "%i2", "%i3", "%i4", "%i5", "%fp", "%i7", - "%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7", - "%f8", "%f9", "%f10", "%f11", "%f12", "%f13", "%f14", "%f15", - "%f16", "%f17", "%f18", "%f19", "%f20", "%f21", "%f22", "%f23", - "%f24", "%f25", "%f26", "%f27", "%f28", "%f29", "%f30", "%f31", - "%f32", "%f33", "%f34", "%f35", "%f36", "%f37", "%f38", "%f39", - "%f40", "%f41", "%f42", "%f43", "%f44", "%f45", "%f46", "%f47", - "%f48", "%f49", "%f50", "%f51", "%f52", "%f53", "%f54", "%f55", - "%f56", "%f57", "%f58", "%f59", "%f60", "%f61", "%f62", "%f63", -}; - -/** - * get_arch_regstr() - lookup register name from it's DWARF register number - * @n: the DWARF register number - * - * get_arch_regstr() returns the name of the register in struct - * regdwarfnum_table from it's DWARF register number. If the register is not - * found in the table, this returns NULL; - */ -const char *get_arch_regstr(unsigned int n) -{ - return (n < SPARC_MAX_REGS) ? sparc_regs_table[n] : NULL; -} diff --git a/tools/perf/arch/x86/Makefile b/tools/perf/arch/x86/Makefile index 67b4969a6738..a6b6e0a9308a 100644 --- a/tools/perf/arch/x86/Makefile +++ b/tools/perf/arch/x86/Makefile @@ -1,9 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif HAVE_KVM_STAT_SUPPORT := 1 -PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 PERF_HAVE_JITDUMP := 1 ### diff --git a/tools/perf/arch/x86/annotate/instructions.c b/tools/perf/arch/x86/annotate/instructions.c index 5caf5a17f03d..ae94b1f0b9cc 100644 --- a/tools/perf/arch/x86/annotate/instructions.c +++ b/tools/perf/arch/x86/annotate/instructions.c @@ -202,12 +202,13 @@ static int x86__annotate_init(struct arch *arch, char *cpuid) if (x86__cpuid_parse(arch, cpuid)) err = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING; } - + arch->e_machine = EM_X86_64; + arch->e_flags = 0; arch->initialized = true; return err; } -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT static void update_insn_state_x86(struct type_state *state, struct data_loc_info *dloc, Dwarf_Die *cu_die, struct disasm_line *dl) diff --git a/tools/perf/arch/x86/tests/intel-cqm.c b/tools/perf/arch/x86/tests/intel-cqm.c deleted file mode 100644 index 360a082fc928..000000000000 --- a/tools/perf/arch/x86/tests/intel-cqm.c +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "tests/tests.h" -#include "cloexec.h" -#include "debug.h" -#include "evlist.h" -#include "evsel.h" -#include "arch-tests.h" -#include <internal/lib.h> // page_size - -#include <signal.h> -#include <sys/mman.h> -#include <sys/wait.h> -#include <errno.h> -#include <string.h> - -static pid_t spawn(void) -{ - pid_t pid; - - pid = fork(); - if (pid) - return pid; - - while(1) - sleep(5); - return 0; -} - -/* - * Create an event group that contains both a sampled hardware - * (cpu-cycles) and software (intel_cqm/llc_occupancy/) event. We then - * wait for the hardware perf counter to overflow and generate a PMI, - * which triggers an event read for both of the events in the group. - * - * Since reading Intel CQM event counters requires sending SMP IPIs, the - * CQM pmu needs to handle the above situation gracefully, and return - * the last read counter value to avoid triggering a WARN_ON_ONCE() in - * smp_call_function_many() caused by sending IPIs from NMI context. - */ -int test__intel_cqm_count_nmi_context(struct test_suite *test __maybe_unused, int subtest __maybe_unused) -{ - struct evlist *evlist = NULL; - struct evsel *evsel = NULL; - struct perf_event_attr pe; - int i, fd[2], flag, ret; - size_t mmap_len; - void *event; - pid_t pid; - int err = TEST_FAIL; - - flag = perf_event_open_cloexec_flag(); - - evlist = evlist__new(); - if (!evlist) { - pr_debug("evlist__new failed\n"); - return TEST_FAIL; - } - - ret = parse_event(evlist, "intel_cqm/llc_occupancy/"); - if (ret) { - pr_debug("parse_events failed, is \"intel_cqm/llc_occupancy/\" available?\n"); - err = TEST_SKIP; - goto out; - } - - evsel = evlist__first(evlist); - if (!evsel) { - pr_debug("evlist__first failed\n"); - goto out; - } - - memset(&pe, 0, sizeof(pe)); - pe.size = sizeof(pe); - - pe.type = PERF_TYPE_HARDWARE; - pe.config = PERF_COUNT_HW_CPU_CYCLES; - pe.read_format = PERF_FORMAT_GROUP; - - pe.sample_period = 128; - pe.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_READ; - - pid = spawn(); - - fd[0] = sys_perf_event_open(&pe, pid, -1, -1, flag); - if (fd[0] < 0) { - pr_debug("failed to open event\n"); - goto out; - } - - memset(&pe, 0, sizeof(pe)); - pe.size = sizeof(pe); - - pe.type = evsel->attr.type; - pe.config = evsel->attr.config; - - fd[1] = sys_perf_event_open(&pe, pid, -1, fd[0], flag); - if (fd[1] < 0) { - pr_debug("failed to open event\n"); - goto out; - } - - /* - * Pick a power-of-two number of pages + 1 for the meta-data - * page (struct perf_event_mmap_page). See tools/perf/design.txt. - */ - mmap_len = page_size * 65; - - event = mmap(NULL, mmap_len, PROT_READ, MAP_SHARED, fd[0], 0); - if (event == (void *)(-1)) { - pr_debug("failed to mmap %d\n", errno); - goto out; - } - - sleep(1); - - err = TEST_OK; - - munmap(event, mmap_len); - - for (i = 0; i < 2; i++) - close(fd[i]); - - kill(pid, SIGKILL); - wait(NULL); -out: - evlist__delete(evlist); - return err; -} diff --git a/tools/perf/arch/x86/tests/intel-pt-test.c b/tools/perf/arch/x86/tests/intel-pt-test.c index 09d61fa736e3..b217ed67cd4e 100644 --- a/tools/perf/arch/x86/tests/intel-pt-test.c +++ b/tools/perf/arch/x86/tests/intel-pt-test.c @@ -375,7 +375,7 @@ static int get_pt_caps(int cpu, struct pt_caps *caps) return 0; } -static bool is_hydrid(void) +static bool is_hybrid(void) { unsigned int eax, ebx, ecx, edx = 0; bool result; @@ -441,7 +441,7 @@ int test__intel_pt_hybrid_compat(struct test_suite *test, int subtest) int ret = TEST_OK; int cpu; - if (!is_hydrid()) { + if (!is_hybrid()) { test->test_cases[subtest].skip_reason = "not hybrid"; return TEST_SKIP; } diff --git a/tools/perf/arch/x86/util/Build b/tools/perf/arch/x86/util/Build index 2607ed5c4296..848327378694 100644 --- a/tools/perf/arch/x86/util/Build +++ b/tools/perf/arch/x86/util/Build @@ -10,10 +10,6 @@ perf-util-y += evlist.o perf-util-y += mem-events.o perf-util-y += evsel.o perf-util-y += iostat.o -perf-util-y += env.o - -perf-util-$(CONFIG_DWARF) += dwarf-regs.o -perf-util-$(CONFIG_BPF_PROLOGUE) += dwarf-regs.o perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/x86/util/auxtrace.c b/tools/perf/arch/x86/util/auxtrace.c index 354780ff1605..ecbf61a7eb3a 100644 --- a/tools/perf/arch/x86/util/auxtrace.c +++ b/tools/perf/arch/x86/util/auxtrace.c @@ -55,11 +55,12 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, int *err) { char buffer[64]; + struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); int ret; *err = 0; - ret = get_cpuid(buffer, sizeof(buffer)); + ret = get_cpuid(buffer, sizeof(buffer), cpu); if (ret) { *err = ret; return NULL; diff --git a/tools/perf/arch/x86/util/dwarf-regs.c b/tools/perf/arch/x86/util/dwarf-regs.c deleted file mode 100644 index 399c4a0a29d8..000000000000 --- a/tools/perf/arch/x86/util/dwarf-regs.c +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. - * Extracted from probe-finder.c - * - * Written by Masami Hiramatsu <mhiramat@redhat.com> - */ - -#include <stddef.h> -#include <errno.h> /* for EINVAL */ -#include <string.h> /* for strcmp */ -#include <linux/ptrace.h> /* for struct pt_regs */ -#include <linux/kernel.h> /* for offsetof */ -#include <dwarf-regs.h> - -/* - * See arch/x86/kernel/ptrace.c. - * Different from it: - * - * - Since struct pt_regs is defined differently for user and kernel, - * but we want to use 'ax, bx' instead of 'rax, rbx' (which is struct - * field name of user's pt_regs), we make REG_OFFSET_NAME to accept - * both string name and reg field name. - * - * - Since accessing x86_32's pt_regs from x86_64 building is difficult - * and vise versa, we simply fill offset with -1, so - * get_arch_regstr() still works but regs_query_register_offset() - * returns error. - * The only inconvenience caused by it now is that we are not allowed - * to generate BPF prologue for a x86_64 kernel if perf is built for - * x86_32. This is really a rare usecase. - * - * - Order is different from kernel's ptrace.c for get_arch_regstr(). Use - * the order defined by dwarf. - */ - -struct pt_regs_offset { - const char *name; - int offset; -}; - -#define REG_OFFSET_END {.name = NULL, .offset = 0} - -#ifdef __x86_64__ -# define REG_OFFSET_NAME_64(n, r) {.name = n, .offset = offsetof(struct pt_regs, r)} -# define REG_OFFSET_NAME_32(n, r) {.name = n, .offset = -1} -#else -# define REG_OFFSET_NAME_64(n, r) {.name = n, .offset = -1} -# define REG_OFFSET_NAME_32(n, r) {.name = n, .offset = offsetof(struct pt_regs, r)} -#endif - -/* TODO: switching by dwarf address size */ -#ifndef __x86_64__ -static const struct pt_regs_offset x86_32_regoffset_table[] = { - REG_OFFSET_NAME_32("%ax", eax), - REG_OFFSET_NAME_32("%cx", ecx), - REG_OFFSET_NAME_32("%dx", edx), - REG_OFFSET_NAME_32("%bx", ebx), - REG_OFFSET_NAME_32("$stack", esp), /* Stack address instead of %sp */ - REG_OFFSET_NAME_32("%bp", ebp), - REG_OFFSET_NAME_32("%si", esi), - REG_OFFSET_NAME_32("%di", edi), - REG_OFFSET_END, -}; - -#define regoffset_table x86_32_regoffset_table -#else -static const struct pt_regs_offset x86_64_regoffset_table[] = { - REG_OFFSET_NAME_64("%ax", rax), - REG_OFFSET_NAME_64("%dx", rdx), - REG_OFFSET_NAME_64("%cx", rcx), - REG_OFFSET_NAME_64("%bx", rbx), - REG_OFFSET_NAME_64("%si", rsi), - REG_OFFSET_NAME_64("%di", rdi), - REG_OFFSET_NAME_64("%bp", rbp), - REG_OFFSET_NAME_64("%sp", rsp), - REG_OFFSET_NAME_64("%r8", r8), - REG_OFFSET_NAME_64("%r9", r9), - REG_OFFSET_NAME_64("%r10", r10), - REG_OFFSET_NAME_64("%r11", r11), - REG_OFFSET_NAME_64("%r12", r12), - REG_OFFSET_NAME_64("%r13", r13), - REG_OFFSET_NAME_64("%r14", r14), - REG_OFFSET_NAME_64("%r15", r15), - REG_OFFSET_END, -}; - -#define regoffset_table x86_64_regoffset_table -#endif - -/* Minus 1 for the ending REG_OFFSET_END */ -#define ARCH_MAX_REGS ((sizeof(regoffset_table) / sizeof(regoffset_table[0])) - 1) - -/* Return architecture dependent register string (for kprobe-tracer) */ -const char *get_arch_regstr(unsigned int n) -{ - return (n < ARCH_MAX_REGS) ? regoffset_table[n].name : NULL; -} - -/* Reuse code from arch/x86/kernel/ptrace.c */ -/** - * regs_query_register_offset() - query register offset from its name - * @name: the name of a register - * - * regs_query_register_offset() returns the offset of a register in struct - * pt_regs from its name. If the name is invalid, this returns -EINVAL; - */ -int regs_query_register_offset(const char *name) -{ - const struct pt_regs_offset *roff; - for (roff = regoffset_table; roff->name != NULL; roff++) - if (!strcmp(roff->name, name)) - return roff->offset; - return -EINVAL; -} - -struct dwarf_regs_idx { - const char *name; - int idx; -}; - -static const struct dwarf_regs_idx x86_regidx_table[] = { - { "rax", 0 }, { "eax", 0 }, { "ax", 0 }, { "al", 0 }, - { "rdx", 1 }, { "edx", 1 }, { "dx", 1 }, { "dl", 1 }, - { "rcx", 2 }, { "ecx", 2 }, { "cx", 2 }, { "cl", 2 }, - { "rbx", 3 }, { "edx", 3 }, { "bx", 3 }, { "bl", 3 }, - { "rsi", 4 }, { "esi", 4 }, { "si", 4 }, { "sil", 4 }, - { "rdi", 5 }, { "edi", 5 }, { "di", 5 }, { "dil", 5 }, - { "rbp", 6 }, { "ebp", 6 }, { "bp", 6 }, { "bpl", 6 }, - { "rsp", 7 }, { "esp", 7 }, { "sp", 7 }, { "spl", 7 }, - { "r8", 8 }, { "r8d", 8 }, { "r8w", 8 }, { "r8b", 8 }, - { "r9", 9 }, { "r9d", 9 }, { "r9w", 9 }, { "r9b", 9 }, - { "r10", 10 }, { "r10d", 10 }, { "r10w", 10 }, { "r10b", 10 }, - { "r11", 11 }, { "r11d", 11 }, { "r11w", 11 }, { "r11b", 11 }, - { "r12", 12 }, { "r12d", 12 }, { "r12w", 12 }, { "r12b", 12 }, - { "r13", 13 }, { "r13d", 13 }, { "r13w", 13 }, { "r13b", 13 }, - { "r14", 14 }, { "r14d", 14 }, { "r14w", 14 }, { "r14b", 14 }, - { "r15", 15 }, { "r15d", 15 }, { "r15w", 15 }, { "r15b", 15 }, - { "rip", DWARF_REG_PC }, -}; - -int get_arch_regnum(const char *name) -{ - unsigned int i; - - if (*name != '%') - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(x86_regidx_table); i++) - if (!strcmp(x86_regidx_table[i].name, name + 1)) - return x86_regidx_table[i].idx; - return -ENOENT; -} diff --git a/tools/perf/arch/x86/util/env.c b/tools/perf/arch/x86/util/env.c deleted file mode 100644 index 3e537ffb1353..000000000000 --- a/tools/perf/arch/x86/util/env.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "linux/string.h" -#include "util/env.h" -#include "env.h" - -bool x86__is_amd_cpu(void) -{ - struct perf_env env = { .total_mem = 0, }; - static int is_amd; /* 0: Uninitialized, 1: Yes, -1: No */ - - if (is_amd) - goto ret; - - perf_env__cpuid(&env); - is_amd = env.cpuid && strstarts(env.cpuid, "AuthenticAMD") ? 1 : -1; - perf_env__exit(&env); -ret: - return is_amd >= 1 ? true : false; -} diff --git a/tools/perf/arch/x86/util/env.h b/tools/perf/arch/x86/util/env.h deleted file mode 100644 index d78f080b6b3f..000000000000 --- a/tools/perf/arch/x86/util/env.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _X86_ENV_H -#define _X86_ENV_H - -bool x86__is_amd_cpu(void); - -#endif /* _X86_ENV_H */ diff --git a/tools/perf/arch/x86/util/evlist.c b/tools/perf/arch/x86/util/evlist.c index cebdd483149e..447a734e591c 100644 --- a/tools/perf/arch/x86/util/evlist.c +++ b/tools/perf/arch/x86/util/evlist.c @@ -1,91 +1,86 @@ // SPDX-License-Identifier: GPL-2.0 -#include <stdio.h> -#include "util/pmu.h" -#include "util/pmus.h" -#include "util/evlist.h" -#include "util/parse-events.h" -#include "util/event.h" +#include <string.h> +#include "../../../util/evlist.h" +#include "../../../util/evsel.h" #include "topdown.h" #include "evsel.h" -static int ___evlist__add_default_attrs(struct evlist *evlist, - struct perf_event_attr *attrs, - size_t nr_attrs) -{ - LIST_HEAD(head); - size_t i = 0; - - for (i = 0; i < nr_attrs; i++) - event_attr_init(attrs + i); - - if (perf_pmus__num_core_pmus() == 1) - return evlist__add_attrs(evlist, attrs, nr_attrs); - - for (i = 0; i < nr_attrs; i++) { - struct perf_pmu *pmu = NULL; - - if (attrs[i].type == PERF_TYPE_SOFTWARE) { - struct evsel *evsel = evsel__new(attrs + i); - - if (evsel == NULL) - goto out_delete_partial_list; - list_add_tail(&evsel->core.node, &head); - continue; - } - - while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { - struct perf_cpu_map *cpus; - struct evsel *evsel; - - evsel = evsel__new(attrs + i); - if (evsel == NULL) - goto out_delete_partial_list; - evsel->core.attr.config |= (__u64)pmu->type << PERF_PMU_TYPE_SHIFT; - cpus = perf_cpu_map__get(pmu->cpus); - evsel->core.cpus = cpus; - evsel->core.own_cpus = perf_cpu_map__get(cpus); - evsel->pmu_name = strdup(pmu->name); - list_add_tail(&evsel->core.node, &head); - } - } - - evlist__splice_list_tail(evlist, &head); - - return 0; - -out_delete_partial_list: - { - struct evsel *evsel, *n; - - __evlist__for_each_entry_safe(&head, n, evsel) - evsel__delete(evsel); - } - return -1; -} - -int arch_evlist__add_default_attrs(struct evlist *evlist, - struct perf_event_attr *attrs, - size_t nr_attrs) -{ - if (!nr_attrs) - return 0; - - return ___evlist__add_default_attrs(evlist, attrs, nr_attrs); -} - int arch_evlist__cmp(const struct evsel *lhs, const struct evsel *rhs) { + /* + * Currently the following topdown events sequence are supported to + * move and regroup correctly. + * + * a. all events in a group + * perf stat -e "{instructions,topdown-retiring,slots}" -C0 sleep 1 + * WARNING: events were regrouped to match PMUs + * Performance counter stats for 'CPU(s) 0': + * 15,066,240 slots + * 1,899,760 instructions + * 2,126,998 topdown-retiring + * b. all events not in a group + * perf stat -e "instructions,topdown-retiring,slots" -C0 sleep 1 + * WARNING: events were regrouped to match PMUs + * Performance counter stats for 'CPU(s) 0': + * 2,045,561 instructions + * 17,108,370 slots + * 2,281,116 topdown-retiring + * c. slots event in a group but topdown metrics events outside the group + * perf stat -e "{instructions,slots},topdown-retiring" -C0 sleep 1 + * WARNING: events were regrouped to match PMUs + * Performance counter stats for 'CPU(s) 0': + * 20,323,878 slots + * 2,634,884 instructions + * 3,028,656 topdown-retiring + * d. slots event and topdown metrics events in two groups + * perf stat -e "{instructions,slots},{topdown-retiring}" -C0 sleep 1 + * WARNING: events were regrouped to match PMUs + * Performance counter stats for 'CPU(s) 0': + * 26,319,024 slots + * 2,427,791 instructions + * 2,683,508 topdown-retiring + * + * If slots event and topdown metrics events are not in same group, the + * topdown metrics events must be first event after the slots event group, + * otherwise topdown metrics events can't be regrouped correctly, e.g. + * + * a. perf stat -e "{instructions,slots},cycles,topdown-retiring" -C0 sleep 1 + * WARNING: events were regrouped to match PMUs + * Performance counter stats for 'CPU(s) 0': + * 17,923,134 slots + * 2,154,855 instructions + * 3,015,058 cycles + * <not supported> topdown-retiring + * + * If slots event and topdown metrics events are in two groups, the group which + * has topdown metrics events must contain only the topdown metrics event, + * otherwise topdown metrics event can't be regrouped correctly as well, e.g. + * + * a. perf stat -e "{instructions,slots},{topdown-retiring,cycles}" -C0 sleep 1 + * WARNING: events were regrouped to match PMUs + * Error: + * The sys_perf_event_open() syscall returned with 22 (Invalid argument) for + * event (topdown-retiring) + */ if (topdown_sys_has_perf_metrics() && (arch_evsel__must_be_in_group(lhs) || arch_evsel__must_be_in_group(rhs))) { /* Ensure the topdown slots comes first. */ - if (strcasestr(lhs->name, "slots") && !strcasestr(lhs->name, "uops_retired.slots")) + if (arch_is_topdown_slots(lhs)) return -1; - if (strcasestr(rhs->name, "slots") && !strcasestr(rhs->name, "uops_retired.slots")) + if (arch_is_topdown_slots(rhs)) return 1; - /* Followed by topdown events. */ - if (strcasestr(lhs->name, "topdown") && !strcasestr(rhs->name, "topdown")) + + /* + * Move topdown metrics events forward only when topdown metrics + * events are not in same group with previous slots event. If + * topdown metrics events are already in same group with slots + * event, do nothing. + */ + if (arch_is_topdown_metrics(lhs) && !arch_is_topdown_metrics(rhs) && + lhs->core.leader != rhs->core.leader) return -1; - if (!strcasestr(lhs->name, "topdown") && strcasestr(rhs->name, "topdown")) + if (!arch_is_topdown_metrics(lhs) && arch_is_topdown_metrics(rhs) && + lhs->core.leader != rhs->core.leader) return 1; } diff --git a/tools/perf/arch/x86/util/evsel.c b/tools/perf/arch/x86/util/evsel.c index 090d0f371891..3dd29ba2c23b 100644 --- a/tools/perf/arch/x86/util/evsel.c +++ b/tools/perf/arch/x86/util/evsel.c @@ -6,6 +6,7 @@ #include "util/pmu.h" #include "util/pmus.h" #include "linux/string.h" +#include "topdown.h" #include "evsel.h" #include "util/debug.h" #include "env.h" @@ -21,7 +22,8 @@ void arch_evsel__set_sample_weight(struct evsel *evsel) /* Check whether the evsel's PMU supports the perf metrics */ bool evsel__sys_has_perf_metrics(const struct evsel *evsel) { - const char *pmu_name = evsel->pmu_name ? evsel->pmu_name : "cpu"; + struct perf_pmu *pmu; + u32 type = evsel->core.attr.type; /* * The PERF_TYPE_RAW type is the core PMU type, e.g., "cpu" PMU @@ -31,11 +33,31 @@ bool evsel__sys_has_perf_metrics(const struct evsel *evsel) * Checking both the PERF_TYPE_RAW type and the slots event * should be good enough to detect the perf metrics feature. */ - if ((evsel->core.attr.type == PERF_TYPE_RAW) && - perf_pmus__have_event(pmu_name, "slots")) - return true; +again: + switch (type) { + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + type = evsel->core.attr.config >> PERF_PMU_TYPE_SHIFT; + if (type) + goto again; + break; + case PERF_TYPE_RAW: + break; + default: + return false; + } + + pmu = evsel->pmu; + if (pmu && perf_pmu__is_fake(pmu)) + pmu = NULL; - return false; + if (!pmu) { + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + if (pmu->type == PERF_TYPE_RAW) + break; + } + } + return pmu && perf_pmu__have_event(pmu, "slots"); } bool arch_evsel__must_be_in_group(const struct evsel *evsel) @@ -44,7 +66,7 @@ bool arch_evsel__must_be_in_group(const struct evsel *evsel) strcasestr(evsel->name, "uops_retired.slots")) return false; - return strcasestr(evsel->name, "topdown") || strcasestr(evsel->name, "slots"); + return arch_is_topdown_metrics(evsel) || arch_is_topdown_slots(evsel); } int arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size) @@ -63,7 +85,7 @@ int arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size) return scnprintf(bf, size, "%s", event_name); return scnprintf(bf, size, "%s/%s/", - evsel->pmu_name ? evsel->pmu_name : "cpu", + evsel->pmu ? evsel->pmu->name : "cpu", event_name); } @@ -108,7 +130,7 @@ int arch_evsel__open_strerror(struct evsel *evsel, char *msg, size_t size) return 0; if (!evsel->core.attr.precise_ip && - !(evsel->pmu_name && !strncmp(evsel->pmu_name, "ibs", 3))) + !(evsel->pmu && !strncmp(evsel->pmu->name, "ibs", 3))) return 0; /* More verbose IBS errors. */ diff --git a/tools/perf/arch/x86/util/header.c b/tools/perf/arch/x86/util/header.c index a51444a77a5f..412977f8aa83 100644 --- a/tools/perf/arch/x86/util/header.c +++ b/tools/perf/arch/x86/util/header.c @@ -58,13 +58,12 @@ __get_cpuid(char *buffer, size_t sz, const char *fmt) } int -get_cpuid(char *buffer, size_t sz) +get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu __maybe_unused) { return __get_cpuid(buffer, sz, "%s,%u,%u,%u$"); } -char * -get_cpuid_str(struct perf_pmu *pmu __maybe_unused) +char *get_cpuid_str(struct perf_cpu cpu __maybe_unused) { char *buf = malloc(128); diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index ea510a7486b1..8f235d8b67b6 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -75,7 +75,8 @@ static int intel_pt_parse_terms_with_default(const struct perf_pmu *pmu, goto out_free; attr.config = *config; - err = perf_pmu__config_terms(pmu, &attr, &terms, /*zero=*/true, /*err=*/NULL); + err = perf_pmu__config_terms(pmu, &attr, &terms, /*zero=*/true, /*apply_hardcoded=*/false, + /*err=*/NULL); if (err) goto out_free; diff --git a/tools/perf/arch/x86/util/iostat.c b/tools/perf/arch/x86/util/iostat.c index df7b5dfcc26a..366b44d0bb7e 100644 --- a/tools/perf/arch/x86/util/iostat.c +++ b/tools/perf/arch/x86/util/iostat.c @@ -444,7 +444,7 @@ void iostat_print_metric(struct perf_stat_config *config, struct evsel *evsel, iostat_value = (count->val - prev_count_val) / ((double) count->run / count->ena); } - out->print_metric(config, out->ctx, NULL, "%8.0f", iostat_metric, + out->print_metric(config, out->ctx, METRIC_THRESHOLD_UNKNOWN, "%8.0f", iostat_metric, iostat_value / (256 * 1024)); } diff --git a/tools/perf/arch/x86/util/pmu.c b/tools/perf/arch/x86/util/pmu.c index c3d89d6ba1bf..e0060dac2a9f 100644 --- a/tools/perf/arch/x86/util/pmu.c +++ b/tools/perf/arch/x86/util/pmu.c @@ -16,7 +16,7 @@ #include "../../../util/fncache.h" #include "../../../util/pmus.h" #include "mem-events.h" -#include "env.h" +#include "util/env.h" void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) { diff --git a/tools/perf/arch/x86/util/topdown.c b/tools/perf/arch/x86/util/topdown.c index 3f9a267d4501..f63747d0abdf 100644 --- a/tools/perf/arch/x86/util/topdown.c +++ b/tools/perf/arch/x86/util/topdown.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include "api/fs/fs.h" #include "util/evsel.h" +#include "util/evlist.h" #include "util/pmu.h" #include "util/pmus.h" #include "util/topdown.h" @@ -32,6 +33,31 @@ bool topdown_sys_has_perf_metrics(void) } #define TOPDOWN_SLOTS 0x0400 +bool arch_is_topdown_slots(const struct evsel *evsel) +{ + if (evsel->core.attr.config == TOPDOWN_SLOTS) + return true; + + return false; +} + +bool arch_is_topdown_metrics(const struct evsel *evsel) +{ + int config = evsel->core.attr.config; + const char *name_from_config; + struct perf_pmu *pmu; + + /* All topdown events have an event code of 0. */ + if ((config & 0xFF) != 0) + return false; + + pmu = evsel__find_pmu(evsel); + if (!pmu || !pmu->is_core) + return false; + + name_from_config = perf_pmu__name_from_config(pmu, config); + return name_from_config && strcasestr(name_from_config, "topdown"); +} /* * Check whether a topdown group supports sample-read. @@ -41,11 +67,24 @@ bool topdown_sys_has_perf_metrics(void) */ bool arch_topdown_sample_read(struct evsel *leader) { + struct evsel *evsel; + if (!evsel__sys_has_perf_metrics(leader)) return false; - if (leader->core.attr.config == TOPDOWN_SLOTS) - return true; + if (!arch_is_topdown_slots(leader)) + return false; + + /* + * If slots event as leader event but no topdown metric events + * in group, slots event should still sample as leader. + */ + evlist__for_each_entry(leader->evlist, evsel) { + if (evsel->core.leader != leader->core.leader) + return false; + if (evsel != leader && arch_is_topdown_metrics(evsel)) + return true; + } return false; } diff --git a/tools/perf/arch/x86/util/topdown.h b/tools/perf/arch/x86/util/topdown.h index 46bf9273e572..1bae9b1822d7 100644 --- a/tools/perf/arch/x86/util/topdown.h +++ b/tools/perf/arch/x86/util/topdown.h @@ -3,5 +3,7 @@ #define _TOPDOWN_H 1 bool topdown_sys_has_perf_metrics(void); +bool arch_is_topdown_slots(const struct evsel *evsel); +bool arch_is_topdown_metrics(const struct evsel *evsel); #endif diff --git a/tools/perf/arch/x86/util/tsc.c b/tools/perf/arch/x86/util/tsc.c index e2d6cfe21057..3a439e4b12d2 100644 --- a/tools/perf/arch/x86/util/tsc.c +++ b/tools/perf/arch/x86/util/tsc.c @@ -24,9 +24,9 @@ u64 rdtsc(void) * ... * will return 3000000000. */ -static double cpuinfo_tsc_freq(void) +static u64 cpuinfo_tsc_freq(void) { - double result = 0; + u64 result = 0; FILE *cpuinfo; char *line = NULL; size_t len = 0; @@ -34,20 +34,22 @@ static double cpuinfo_tsc_freq(void) cpuinfo = fopen("/proc/cpuinfo", "r"); if (!cpuinfo) { pr_err("Failed to read /proc/cpuinfo for TSC frequency\n"); - return NAN; + return 0; } while (getline(&line, &len, cpuinfo) > 0) { if (!strncmp(line, "model name", 10)) { char *pos = strstr(line + 11, " @ "); + double float_result; - if (pos && sscanf(pos, " @ %lfGHz", &result) == 1) { - result *= 1000000000; + if (pos && sscanf(pos, " @ %lfGHz", &float_result) == 1) { + float_result *= 1000000000; + result = (u64)float_result; goto out; } } } out: - if (fpclassify(result) == FP_ZERO) + if (result == 0) pr_err("Failed to find TSC frequency in /proc/cpuinfo\n"); free(line); @@ -55,7 +57,7 @@ out: return result; } -double arch_get_tsc_freq(void) +u64 arch_get_tsc_freq(void) { unsigned int a, b, c, d, lvl; static bool cached; @@ -86,6 +88,6 @@ double arch_get_tsc_freq(void) return tsc; } - tsc = (double)c * (double)b / (double)a; + tsc = (u64)c * (u64)b / (u64)a; return tsc; } diff --git a/tools/perf/arch/xtensa/Build b/tools/perf/arch/xtensa/Build deleted file mode 100644 index e63eabc2c8f4..000000000000 --- a/tools/perf/arch/xtensa/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-y += util/ diff --git a/tools/perf/arch/xtensa/Makefile b/tools/perf/arch/xtensa/Makefile deleted file mode 100644 index 88c08eed9c7b..000000000000 --- a/tools/perf/arch/xtensa/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ifndef NO_DWARF -PERF_HAVE_DWARF_REGS := 1 -endif diff --git a/tools/perf/arch/xtensa/util/Build b/tools/perf/arch/xtensa/util/Build deleted file mode 100644 index e813e618954b..000000000000 --- a/tools/perf/arch/xtensa/util/Build +++ /dev/null @@ -1 +0,0 @@ -perf-$(CONFIG_DWARF) += dwarf-regs.o diff --git a/tools/perf/arch/xtensa/util/dwarf-regs.c b/tools/perf/arch/xtensa/util/dwarf-regs.c deleted file mode 100644 index 12f5457300f5..000000000000 --- a/tools/perf/arch/xtensa/util/dwarf-regs.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Mapping of DWARF debug register numbers into register names. - * - * Copyright (c) 2015 Cadence Design Systems Inc. - */ - -#include <stddef.h> -#include <dwarf-regs.h> - -#define XTENSA_MAX_REGS 16 - -const char *xtensa_regs_table[XTENSA_MAX_REGS] = { - "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", - "a8", "a9", "a10", "a11", "a12", "a13", "a14", "a15", -}; - -const char *get_arch_regstr(unsigned int n) -{ - return n < XTENSA_MAX_REGS ? xtensa_regs_table[n] : NULL; -} diff --git a/tools/perf/bench/numa.c b/tools/perf/bench/numa.c index 1fbd7c947abc..19be2aaf4dc0 100644 --- a/tools/perf/bench/numa.c +++ b/tools/perf/bench/numa.c @@ -27,6 +27,7 @@ #include <sys/resource.h> #include <sys/wait.h> #include <sys/prctl.h> +#include <sys/stat.h> #include <sys/types.h> #include <linux/kernel.h> #include <linux/time64.h> @@ -35,6 +36,7 @@ #include "../util/header.h" #include "../util/mutex.h" +#include <api/fs/fs.h> #include <numa.h> #include <numaif.h> @@ -533,6 +535,57 @@ static int parse_cpu_list(const char *arg) return 0; } +/* + * Check whether a CPU is online + * + * Returns: + * 1 -> if CPU is online + * 0 -> if CPU is offline + * -1 -> error case + */ +static int is_cpu_online(unsigned int cpu) +{ + char *str; + size_t strlen; + char buf[256]; + int status = -1; + struct stat statbuf; + + snprintf(buf, sizeof(buf), + "/sys/devices/system/cpu/cpu%d", cpu); + if (stat(buf, &statbuf) != 0) + return 0; + + /* + * Check if /sys/devices/system/cpu/cpux/online file + * exists. Some cases cpu0 won't have online file since + * it is not expected to be turned off generally. + * In kernels without CONFIG_HOTPLUG_CPU, this + * file won't exist + */ + snprintf(buf, sizeof(buf), + "/sys/devices/system/cpu/cpu%d/online", cpu); + if (stat(buf, &statbuf) != 0) + return 1; + + /* + * Read online file using sysfs__read_str. + * If read or open fails, return -1. + * If read succeeds, return value from file + * which gets stored in "str" + */ + snprintf(buf, sizeof(buf), + "devices/system/cpu/cpu%d/online", cpu); + + if (sysfs__read_str(buf, &str, &strlen) < 0) + return status; + + status = atoi(str); + + free(str); + return status; +} + static int parse_setup_cpu_list(void) { struct thread_data *td; diff --git a/tools/perf/bench/sched-pipe.c b/tools/perf/bench/sched-pipe.c index 3af6d3c55aba..e2562677df96 100644 --- a/tools/perf/bench/sched-pipe.c +++ b/tools/perf/bench/sched-pipe.c @@ -23,6 +23,7 @@ #include <errno.h> #include <fcntl.h> #include <assert.h> +#include <sys/epoll.h> #include <sys/time.h> #include <sys/types.h> #include <sys/syscall.h> @@ -34,6 +35,8 @@ struct thread_data { int nr; int pipe_read; int pipe_write; + struct epoll_event epoll_ev; + int epoll_fd; bool cgroup_failed; pthread_t pthread; }; @@ -44,6 +47,7 @@ static int loops = LOOPS_DEFAULT; /* Use processes by default: */ static bool threaded; +static bool nonblocking; static char *cgrp_names[2]; static struct cgroup *cgrps[2]; @@ -81,6 +85,7 @@ out: } static const struct option options[] = { + OPT_BOOLEAN('n', "nonblocking", &nonblocking, "Use non-blocking operations"), OPT_INTEGER('l', "loop", &loops, "Specify number of loops"), OPT_BOOLEAN('T', "threaded", &threaded, "Specify threads/process based task setup"), OPT_CALLBACK('G', "cgroups", NULL, "SEND,RECV", @@ -165,11 +170,25 @@ static void exit_cgroup(int nr) free(cgrp_names[nr]); } +static inline int read_pipe(struct thread_data *td) +{ + int ret, m; +retry: + if (nonblocking) { + ret = epoll_wait(td->epoll_fd, &td->epoll_ev, 1, -1); + if (ret < 0) + return ret; + } + ret = read(td->pipe_read, &m, sizeof(int)); + if (nonblocking && ret < 0 && errno == EWOULDBLOCK) + goto retry; + return ret; +} + static void *worker_thread(void *__tdata) { struct thread_data *td = __tdata; - int m = 0, i; - int ret; + int i, ret, m = 0; ret = enter_cgroup(td->nr); if (ret < 0) { @@ -177,16 +196,23 @@ static void *worker_thread(void *__tdata) return NULL; } + if (nonblocking) { + td->epoll_ev.events = EPOLLIN; + td->epoll_fd = epoll_create(1); + BUG_ON(td->epoll_fd < 0); + BUG_ON(epoll_ctl(td->epoll_fd, EPOLL_CTL_ADD, td->pipe_read, &td->epoll_ev) < 0); + } + for (i = 0; i < loops; i++) { if (!td->nr) { - ret = read(td->pipe_read, &m, sizeof(int)); + ret = read_pipe(td); BUG_ON(ret != sizeof(int)); ret = write(td->pipe_write, &m, sizeof(int)); BUG_ON(ret != sizeof(int)); } else { ret = write(td->pipe_write, &m, sizeof(int)); BUG_ON(ret != sizeof(int)); - ret = read(td->pipe_read, &m, sizeof(int)); + ret = read_pipe(td); BUG_ON(ret != sizeof(int)); } } @@ -209,13 +235,16 @@ int bench_sched_pipe(int argc, const char **argv) * discarding returned value of read(), write() * causes error in building environment for perf */ - int __maybe_unused ret, wait_stat; + int __maybe_unused ret, wait_stat, flags = 0; pid_t pid, retpid __maybe_unused; argc = parse_options(argc, argv, options, bench_sched_pipe_usage, 0); - BUG_ON(pipe(pipe_1)); - BUG_ON(pipe(pipe_2)); + if (nonblocking) + flags |= O_NONBLOCK; + + BUG_ON(pipe2(pipe_1, flags)); + BUG_ON(pipe2(pipe_2, flags)); gettimeofday(&start, NULL); diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c index 3dc6197ef3fa..bb87e6e7687d 100644 --- a/tools/perf/builtin-annotate.c +++ b/tools/perf/builtin-annotate.c @@ -840,7 +840,7 @@ int cmd_annotate(int argc, const char **argv) } #endif -#ifndef HAVE_DWARF_GETLOCATIONS_SUPPORT +#ifndef HAVE_LIBDW_SUPPORT if (annotate.data_type) { pr_err("Error: Data type profiling is disabled due to missing DWARF support\n"); return -ENOTSUP; diff --git a/tools/perf/builtin-check.c b/tools/perf/builtin-check.c index 0b76b6e42b78..2346536a5ee1 100644 --- a/tools/perf/builtin-check.c +++ b/tools/perf/builtin-check.c @@ -27,15 +27,15 @@ struct feature_status supported_features[] = { FEATURE_STATUS("bpf", HAVE_LIBBPF_SUPPORT), FEATURE_STATUS("bpf_skeletons", HAVE_BPF_SKEL), FEATURE_STATUS("debuginfod", HAVE_DEBUGINFOD_SUPPORT), - FEATURE_STATUS("dwarf", HAVE_DWARF_SUPPORT), - FEATURE_STATUS("dwarf_getlocations", HAVE_DWARF_GETLOCATIONS_SUPPORT), + FEATURE_STATUS("dwarf", HAVE_LIBDW_SUPPORT), + FEATURE_STATUS("dwarf_getlocations", HAVE_LIBDW_SUPPORT), FEATURE_STATUS("dwarf-unwind", HAVE_DWARF_UNWIND_SUPPORT), FEATURE_STATUS("auxtrace", HAVE_AUXTRACE_SUPPORT), FEATURE_STATUS("libaudit", HAVE_LIBAUDIT_SUPPORT), FEATURE_STATUS("libbfd", HAVE_LIBBFD_SUPPORT), FEATURE_STATUS("libcapstone", HAVE_LIBCAPSTONE_SUPPORT), FEATURE_STATUS("libcrypto", HAVE_LIBCRYPTO_SUPPORT), - FEATURE_STATUS("libdw-dwarf-unwind", HAVE_DWARF_SUPPORT), + FEATURE_STATUS("libdw-dwarf-unwind", HAVE_LIBDW_SUPPORT), FEATURE_STATUS("libelf", HAVE_LIBELF_SUPPORT), FEATURE_STATUS("libnuma", HAVE_LIBNUMA_SUPPORT), FEATURE_STATUS("libopencsd", HAVE_CSTRACE_SUPPORT), diff --git a/tools/perf/builtin-diff.c b/tools/perf/builtin-diff.c index 23326dd20333..82fb7773e03e 100644 --- a/tools/perf/builtin-diff.c +++ b/tools/perf/builtin-diff.c @@ -469,13 +469,13 @@ out: static struct perf_diff pdiff; -static struct evsel *evsel_match(struct evsel *evsel, - struct evlist *evlist) +static struct evsel *evsel_match(struct evsel *evsel, struct evlist *evlist) { struct evsel *e; evlist__for_each_entry(evlist, e) { - if (evsel__match2(evsel, e)) + if ((evsel->core.attr.type == e->core.attr.type) && + (evsel->core.attr.config == e->core.attr.config)) return e; } diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index abcdc49b7a98..272d3c70810e 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -815,7 +815,7 @@ static void display_histogram(int buckets[], bool use_nsec) bar_len = buckets[0] * bar_total / total; printf(" %4d - %-4d %s | %10d | %.*s%*s |\n", - 0, 1, "us", buckets[0], bar_len, bar, bar_total - bar_len, ""); + 0, 1, use_nsec ? "ns" : "us", buckets[0], bar_len, bar, bar_total - bar_len, ""); for (i = 1; i < NUM_BUCKET - 1; i++) { int start = (1 << (i - 1)); diff --git a/tools/perf/builtin-kmem.c b/tools/perf/builtin-kmem.c index a756147e2eec..4d8d94146f8d 100644 --- a/tools/perf/builtin-kmem.c +++ b/tools/perf/builtin-kmem.c @@ -36,7 +36,7 @@ #include <regex.h> #include <linux/ctype.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> static int kmem_slab; static int kmem_page; diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c index 55ea17c5ff02..274568d712d1 100644 --- a/tools/perf/builtin-kvm.c +++ b/tools/perf/builtin-kvm.c @@ -1226,7 +1226,9 @@ static int cpu_isa_config(struct perf_kvm_stat *kvm) int err; if (kvm->live) { - err = get_cpuid(buf, sizeof(buf)); + struct perf_cpu cpu = {-1}; + + err = get_cpuid(buf, sizeof(buf), cpu); if (err != 0) { pr_err("Failed to look up CPU type: %s\n", str_error_r(err, buf, sizeof(buf))); @@ -2147,6 +2149,7 @@ int cmd_kvm(int argc, const char **argv) "buildid-list", "stat", NULL }; const char *kvm_usage[] = { NULL, NULL }; + exclude_GH_default = true; perf_host = 0; perf_guest = 1; diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index c1daf82c9b92..8234410cba4c 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -23,7 +23,7 @@ #include <subcmd/pager.h> #include <subcmd/parse-options.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> #include <errno.h> #include <inttypes.h> diff --git a/tools/perf/builtin-list.c b/tools/perf/builtin-list.c index 65b8cba324be..9e7fdfcdd7ff 100644 --- a/tools/perf/builtin-list.c +++ b/tools/perf/builtin-list.c @@ -19,6 +19,7 @@ #include "util/string2.h" #include "util/strlist.h" #include "util/strbuf.h" +#include "util/tool_pmu.h" #include <subcmd/pager.h> #include <subcmd/parse-options.h> #include <linux/zalloc.h> @@ -112,7 +113,7 @@ static void wordwrap(FILE *fp, const char *s, int start, int max, int corr) } } -static void default_print_event(void *ps, const char *pmu_name, const char *topic, +static void default_print_event(void *ps, const char *topic, const char *pmu_name, const char *event_name, const char *event_alias, const char *scale_unit __maybe_unused, bool deprecated, const char *event_type_desc, @@ -353,7 +354,7 @@ static void fix_escape_fprintf(FILE *fp, struct strbuf *buf, const char *fmt, .. fputs(buf->buf, fp); } -static void json_print_event(void *ps, const char *pmu_name, const char *topic, +static void json_print_event(void *ps, const char *topic, const char *pmu_name, const char *event_name, const char *event_alias, const char *scale_unit, bool deprecated, const char *event_type_desc, @@ -614,9 +615,18 @@ int cmd_list(int argc, const char **argv) event_symbols_hw, PERF_COUNT_HW_MAX); else if (strcmp(argv[i], "sw") == 0 || strcmp(argv[i], "software") == 0) { + char *old_pmu_glob = default_ps.pmu_glob; + print_symbol_events(&print_cb, ps, PERF_TYPE_SOFTWARE, event_symbols_sw, PERF_COUNT_SW_MAX); - print_tool_events(&print_cb, ps); + default_ps.pmu_glob = strdup("tool"); + if (!default_ps.pmu_glob) { + ret = -1; + goto out; + } + perf_pmus__print_pmu_events(&print_cb, ps); + zfree(&default_ps.pmu_glob); + default_ps.pmu_glob = old_pmu_glob; } else if (strcmp(argv[i], "cache") == 0 || strcmp(argv[i], "hwcache") == 0) print_hwcache_events(&print_cb, ps); @@ -664,7 +674,6 @@ int cmd_list(int argc, const char **argv) event_symbols_hw, PERF_COUNT_HW_MAX); print_symbol_events(&print_cb, ps, PERF_TYPE_SOFTWARE, event_symbols_sw, PERF_COUNT_SW_MAX); - print_tool_events(&print_cb, ps); print_hwcache_events(&print_cb, ps); perf_pmus__print_pmu_events(&print_cb, ps); print_tracepoint_events(&print_cb, ps); diff --git a/tools/perf/builtin-probe.c b/tools/perf/builtin-probe.c index 003a3bcebfdf..69800e4d9530 100644 --- a/tools/perf/builtin-probe.c +++ b/tools/perf/builtin-probe.c @@ -229,7 +229,7 @@ static int opt_set_target_ns(const struct option *opt __maybe_unused, /* Command option callbacks */ -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT static int opt_show_lines(const struct option *opt, const char *str, int unset __maybe_unused) { @@ -505,7 +505,7 @@ out: return ret; } -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT #define PROBEDEF_STR \ "[EVENT=]FUNC[@SRC][+OFF|%return|:RL|;PT]|SRC:AL|SRC;PT [[NAME=]ARG ...]" #else @@ -521,7 +521,7 @@ __cmd_probe(int argc, const char **argv) "perf probe [<options>] --add 'PROBEDEF' [--add 'PROBEDEF' ...]", "perf probe [<options>] --del '[GROUP:]EVENT' ...", "perf probe --list [GROUP:]EVENT ...", -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT "perf probe [<options>] --line 'LINEDESC'", "perf probe [<options>] --vars 'PROBEPOINT'", #endif @@ -545,7 +545,7 @@ __cmd_probe(int argc, const char **argv) "\t\tFUNC:\tFunction name\n" "\t\tOFF:\tOffset from function entry (in byte)\n" "\t\t%return:\tPut the probe at function return\n" -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT "\t\tSRC:\tSource code path\n" "\t\tRL:\tRelative line number from function entry.\n" "\t\tAL:\tAbsolute line number in file.\n" @@ -612,11 +612,11 @@ __cmd_probe(int argc, const char **argv) set_option_flag(options, 'd', "del", PARSE_OPT_EXCLUSIVE); set_option_flag(options, 'D', "definition", PARSE_OPT_EXCLUSIVE); set_option_flag(options, 'l', "list", PARSE_OPT_EXCLUSIVE); -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT set_option_flag(options, 'L', "line", PARSE_OPT_EXCLUSIVE); set_option_flag(options, 'V', "vars", PARSE_OPT_EXCLUSIVE); #else -# define set_nobuild(s, l, c) set_option_nobuild(options, s, l, "NO_DWARF=1", c) +# define set_nobuild(s, l, c) set_option_nobuild(options, s, l, "NO_LIBDW=1", c) set_nobuild('L', "line", false); set_nobuild('V', "vars", false); set_nobuild('\0', "externs", false); @@ -694,7 +694,7 @@ __cmd_probe(int argc, const char **argv) if (ret < 0) pr_err_with_code(" Error: Failed to show functions.", ret); return ret; -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT case 'L': ret = show_line_range(¶ms->line_range, params->target, params->nsi, params->uprobes); diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index adbaf80b398c..f83252472921 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -4157,9 +4157,7 @@ int cmd_record(int argc, const char **argv) record.opts.tail_synthesize = true; if (rec->evlist->core.nr_entries == 0) { - bool can_profile_kernel = perf_event_paranoid_check(1); - - err = parse_event(rec->evlist, can_profile_kernel ? "cycles:P" : "cycles:Pu"); + err = parse_event(rec->evlist, "cycles:P"); if (err) goto out; } diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index 5dc17ffee27a..048c91960ba9 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -70,7 +70,7 @@ #include <linux/mman.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif struct report { @@ -455,7 +455,7 @@ static int report__setup_sample_type(struct report *rep) if (!(evlist__combined_branch_type(session->evlist) & PERF_SAMPLE_BRANCH_ANY)) rep->nonany_branch_mode = true; -#if !defined(HAVE_LIBUNWIND_SUPPORT) && !defined(HAVE_DWARF_SUPPORT) +#if !defined(HAVE_LIBUNWIND_SUPPORT) && !defined(HAVE_LIBDW_SUPPORT) if (dwarf_callchain_users) { ui__warning("Please install libunwind or libdw " "development packages during the perf build.\n"); @@ -1271,6 +1271,10 @@ static int process_attr(const struct perf_tool *tool __maybe_unused, return 0; } +#define CALLCHAIN_BRANCH_SORT_ORDER \ + "srcline,symbol,dso,callchain_branch_predicted," \ + "callchain_branch_abort,callchain_branch_cycles" + int cmd_report(int argc, const char **argv) { struct perf_session *session; @@ -1639,7 +1643,7 @@ repeat: symbol_conf.use_callchain = true; callchain_register_param(&callchain_param); if (sort_order == NULL) - sort_order = "srcline,symbol,dso"; + sort_order = CALLCHAIN_BRANCH_SORT_ORDER; } if (report.mem_mode) { @@ -1701,7 +1705,7 @@ repeat: report.data_type = true; annotate_opts.annotate_src = false; -#ifndef HAVE_DWARF_GETLOCATIONS_SUPPORT +#ifndef HAVE_LIBDW_SUPPORT pr_err("Error: Data type profiling is disabled due to missing DWARF support\n"); goto error; #endif diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 5981cc51abc8..7049c60ebf77 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -68,7 +68,6 @@ struct task_desc { struct sched_atom **atoms; pthread_t thread; - sem_t sleep_sem; sem_t ready_for_work; sem_t work_done_sem; @@ -80,12 +79,10 @@ enum sched_event_type { SCHED_EVENT_RUN, SCHED_EVENT_SLEEP, SCHED_EVENT_WAKEUP, - SCHED_EVENT_MIGRATION, }; struct sched_atom { enum sched_event_type type; - int specific_wait; u64 timestamp; u64 duration; unsigned long nr; @@ -228,6 +225,7 @@ struct perf_sched { bool show_wakeups; bool show_next; bool show_migrations; + bool pre_migrations; bool show_state; bool show_prio; u64 skipped_samples; @@ -247,7 +245,9 @@ struct thread_runtime { u64 dt_iowait; /* time between CPU access by iowait (off cpu) */ u64 dt_preempt; /* time between CPU access by preempt (off cpu) */ u64 dt_delay; /* time between wakeup and sched-in */ + u64 dt_pre_mig; /* time between migration and wakeup */ u64 ready_to_run; /* time of wakeup */ + u64 migrated; /* time when a thread is migrated */ struct stats run_stats; u64 total_run_time; @@ -255,6 +255,7 @@ struct thread_runtime { u64 total_iowait_time; u64 total_preempt_time; u64 total_delay_time; + u64 total_pre_mig_time; char last_state; @@ -421,14 +422,13 @@ static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *t wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem)); sem_init(wakee_event->wait_sem, 0, 0); - wakee_event->specific_wait = 1; event->wait_sem = wakee_event->wait_sem; sched->nr_wakeup_events++; } static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *task, - u64 timestamp, const char task_state __maybe_unused) + u64 timestamp) { struct sched_atom *event = get_new_event(task, timestamp); @@ -468,7 +468,7 @@ static struct task_desc *register_pid(struct perf_sched *sched, * every task starts in sleeping state - this gets ignored * if there's no wakeup pointing to this sleep state: */ - add_sched_event_sleep(sched, task, 0, 0); + add_sched_event_sleep(sched, task, 0); sched->pid_to_task[pid] = task; sched->nr_tasks++; @@ -529,8 +529,6 @@ static void perf_sched__process_event(struct perf_sched *sched, ret = sem_post(atom->wait_sem); BUG_ON(ret); break; - case SCHED_EVENT_MIGRATION: - break; default: BUG_ON(1); } @@ -673,7 +671,6 @@ static void create_tasks(struct perf_sched *sched) parms->task = task = sched->tasks[i]; parms->sched = sched; parms->fd = self_open_counters(sched, i); - sem_init(&task->sleep_sem, 0, 0); sem_init(&task->ready_for_work, 0, 0); sem_init(&task->work_done_sem, 0, 0); task->curr_event = 0; @@ -697,7 +694,6 @@ static void destroy_tasks(struct perf_sched *sched) task = sched->tasks[i]; err = pthread_join(task->thread, NULL); BUG_ON(err); - sem_destroy(&task->sleep_sem); sem_destroy(&task->ready_for_work); sem_destroy(&task->work_done_sem); } @@ -751,7 +747,6 @@ static void wait_for_tasks(struct perf_sched *sched) for (i = 0; i < sched->nr_tasks; i++) { task = sched->tasks[i]; - sem_init(&task->sleep_sem, 0, 0); task->curr_event = 0; } } @@ -852,7 +847,6 @@ static int replay_switch_event(struct perf_sched *sched, *next_comm = evsel__strval(evsel, sample, "next_comm"); const u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"), next_pid = evsel__intval(evsel, sample, "next_pid"); - const char prev_state = evsel__taskstate(evsel, sample, "prev_state"); struct task_desc *prev, __maybe_unused *next; u64 timestamp0, timestamp = sample->time; int cpu = sample->cpu; @@ -884,7 +878,7 @@ static int replay_switch_event(struct perf_sched *sched, sched->cpu_last_switched[cpu] = timestamp; add_sched_event_run(sched, prev, timestamp, delta); - add_sched_event_sleep(sched, prev, timestamp, prev_state); + add_sched_event_sleep(sched, prev, timestamp); return 0; } @@ -1749,7 +1743,7 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, } if (sched->map.comp && new_cpu) - color_fprintf(stdout, color, " (CPU %d)", this_cpu); + color_fprintf(stdout, color, " (CPU %d)", this_cpu.cpu); if (proceed != 1) { color_fprintf(stdout, color, "\n"); @@ -2083,14 +2077,15 @@ static void timehist_header(struct perf_sched *sched) printf(" "); } - if (sched->show_prio) { - printf(" %-*s %-*s %9s %9s %9s", - comm_width, "task name", MAX_PRIO_STR_LEN, "prio", - "wait time", "sch delay", "run time"); - } else { - printf(" %-*s %9s %9s %9s", comm_width, - "task name", "wait time", "sch delay", "run time"); - } + printf(" %-*s", comm_width, "task name"); + + if (sched->show_prio) + printf(" %-*s", MAX_PRIO_STR_LEN, "prio"); + + printf(" %9s %9s %9s", "wait time", "sch delay", "run time"); + + if (sched->pre_migrations) + printf(" %9s", "pre-mig time"); if (sched->show_state) printf(" %s", "state"); @@ -2105,17 +2100,15 @@ static void timehist_header(struct perf_sched *sched) if (sched->show_cpu_visual) printf(" %*s ", ncpus, ""); - if (sched->show_prio) { - printf(" %-*s %-*s %9s %9s %9s", - comm_width, "[tid/pid]", MAX_PRIO_STR_LEN, "", - "(msec)", "(msec)", "(msec)"); - } else { - printf(" %-*s %9s %9s %9s", comm_width, - "[tid/pid]", "(msec)", "(msec)", "(msec)"); - } + printf(" %-*s", comm_width, "[tid/pid]"); - if (sched->show_state) - printf(" %5s", ""); + if (sched->show_prio) + printf(" %-*s", MAX_PRIO_STR_LEN, ""); + + printf(" %9s %9s %9s", "(msec)", "(msec)", "(msec)"); + + if (sched->pre_migrations) + printf(" %9s", "(msec)"); printf("\n"); @@ -2127,15 +2120,15 @@ static void timehist_header(struct perf_sched *sched) if (sched->show_cpu_visual) printf(" %.*s ", ncpus, graph_dotted_line); - if (sched->show_prio) { - printf(" %.*s %.*s %.9s %.9s %.9s", - comm_width, graph_dotted_line, MAX_PRIO_STR_LEN, graph_dotted_line, - graph_dotted_line, graph_dotted_line, graph_dotted_line); - } else { - printf(" %.*s %.9s %.9s %.9s", comm_width, - graph_dotted_line, graph_dotted_line, graph_dotted_line, - graph_dotted_line); - } + printf(" %.*s", comm_width, graph_dotted_line); + + if (sched->show_prio) + printf(" %.*s", MAX_PRIO_STR_LEN, graph_dotted_line); + + printf(" %.9s %.9s %.9s", graph_dotted_line, graph_dotted_line, graph_dotted_line); + + if (sched->pre_migrations) + printf(" %.9s", graph_dotted_line); if (sched->show_state) printf(" %.5s", graph_dotted_line); @@ -2190,6 +2183,8 @@ static void timehist_print_sample(struct perf_sched *sched, print_sched_time(tr->dt_delay, 6); print_sched_time(tr->dt_run, 6); + if (sched->pre_migrations) + print_sched_time(tr->dt_pre_mig, 6); if (sched->show_state) printf(" %5c ", thread__tid(thread) == 0 ? 'I' : state); @@ -2227,18 +2222,21 @@ out: * last_time = time of last sched change event for current task * (i.e, time process was last scheduled out) * ready_to_run = time of wakeup for current task + * migrated = time of task migration to another CPU * - * -----|------------|------------|------------|------ - * last ready tprev t + * -----|-------------|-------------|-------------|-------------|----- + * last ready migrated tprev t * time to run * - * |-------- dt_wait --------| - * |- dt_delay -|-- dt_run --| + * |---------------- dt_wait ----------------| + * |--------- dt_delay ---------|-- dt_run --| + * |- dt_pre_mig -| * - * dt_run = run time of current task - * dt_wait = time between last schedule out event for task and tprev - * represents time spent off the cpu - * dt_delay = time between wakeup and schedule-in of task + * dt_run = run time of current task + * dt_wait = time between last schedule out event for task and tprev + * represents time spent off the cpu + * dt_delay = time between wakeup and schedule-in of task + * dt_pre_mig = time between wakeup and migration to another CPU */ static void timehist_update_runtime_stats(struct thread_runtime *r, @@ -2249,6 +2247,7 @@ static void timehist_update_runtime_stats(struct thread_runtime *r, r->dt_iowait = 0; r->dt_preempt = 0; r->dt_run = 0; + r->dt_pre_mig = 0; if (tprev) { r->dt_run = t - tprev; @@ -2257,6 +2256,9 @@ static void timehist_update_runtime_stats(struct thread_runtime *r, pr_debug("time travel: wakeup time for task > previous sched_switch event\n"); else r->dt_delay = tprev - r->ready_to_run; + + if ((r->migrated > r->ready_to_run) && (r->migrated < tprev)) + r->dt_pre_mig = r->migrated - r->ready_to_run; } if (r->last_time > tprev) @@ -2280,6 +2282,7 @@ static void timehist_update_runtime_stats(struct thread_runtime *r, r->total_sleep_time += r->dt_sleep; r->total_iowait_time += r->dt_iowait; r->total_preempt_time += r->dt_preempt; + r->total_pre_mig_time += r->dt_pre_mig; } static bool is_idle_sample(struct perf_sample *sample, @@ -2693,9 +2696,13 @@ static int timehist_migrate_task_event(const struct perf_tool *tool, return -1; tr->migrations++; + tr->migrated = sample->time; /* show migrations if requested */ - timehist_print_migration_event(sched, evsel, sample, machine, thread); + if (sched->show_migrations) { + timehist_print_migration_event(sched, evsel, sample, + machine, thread); + } return 0; } @@ -2846,11 +2853,13 @@ out: /* last state is used to determine where to account wait time */ tr->last_state = state; - /* sched out event for task so reset ready to run time */ + /* sched out event for task so reset ready to run time and migrated time */ if (state == 'R') tr->ready_to_run = t; else tr->ready_to_run = 0; + + tr->migrated = 0; } evsel__save_time(evsel, sample->time, sample->cpu); @@ -3290,8 +3299,8 @@ static int perf_sched__timehist(struct perf_sched *sched) goto out; } - if (sched->show_migrations && - perf_session__set_tracepoints_handlers(session, migrate_handlers)) + if ((sched->show_migrations || sched->pre_migrations) && + perf_session__set_tracepoints_handlers(session, migrate_handlers)) goto out; /* pre-allocate struct for per-CPU idle stats */ @@ -3833,6 +3842,7 @@ int cmd_sched(int argc, const char **argv) OPT_BOOLEAN(0, "show-prio", &sched.show_prio, "Show task priority"), OPT_STRING(0, "prio", &sched.prio_str, "prio", "analyze events only for given task priority(ies)"), + OPT_BOOLEAN('P', "pre-migrations", &sched.pre_migrations, "Show pre-migration wait time"), OPT_PARENT(sched_options) }; diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index a644787fa9e1..9e47905f75a6 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -67,7 +67,7 @@ #include <linux/ctype.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif static char const *script_name; @@ -1728,6 +1728,7 @@ static struct { {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "tr end"}, {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vmentry"}, {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vmexit"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_BRANCH_MISS, "br miss"}, {0, NULL} }; @@ -2136,11 +2137,11 @@ struct metric_ctx { }; static void script_print_metric(struct perf_stat_config *config __maybe_unused, - void *ctx, const char *color, - const char *fmt, - const char *unit, double val) + void *ctx, enum metric_threshold_classify thresh, + const char *fmt, const char *unit, double val) { struct metric_ctx *mctx = ctx; + const char *color = metric_threshold_classify__color(thresh); if (!fmt) return; diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 689a3d43c258..fdf5172646a5 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -46,6 +46,7 @@ #include "util/parse-events.h" #include "util/pmus.h" #include "util/pmu.h" +#include "util/tool_pmu.h" #include "util/event.h" #include "util/evlist.h" #include "util/evsel.h" @@ -294,14 +295,14 @@ static int read_single_counter(struct evsel *counter, int cpu_map_idx, int threa * terminates. Use the wait4 values in that case. */ if (err && cpu_map_idx == 0 && - (evsel__tool_event(counter) == PERF_TOOL_USER_TIME || - evsel__tool_event(counter) == PERF_TOOL_SYSTEM_TIME)) { + (evsel__tool_event(counter) == TOOL_PMU__EVENT_USER_TIME || + evsel__tool_event(counter) == TOOL_PMU__EVENT_SYSTEM_TIME)) { u64 val, *start_time; struct perf_counts_values *count = perf_counts(counter->counts, cpu_map_idx, thread); start_time = xyarray__entry(counter->start_times, cpu_map_idx, thread); - if (evsel__tool_event(counter) == PERF_TOOL_USER_TIME) + if (evsel__tool_event(counter) == TOOL_PMU__EVENT_USER_TIME) val = ru_stats.ru_utime_usec_stat.mean; else val = ru_stats.ru_stime_usec_stat.mean; @@ -639,8 +640,7 @@ static enum counter_recovery stat_handle_error(struct evsel *counter) * (behavior changed with commit b0a873e). */ if (errno == EINVAL || errno == ENOSYS || - errno == ENOENT || errno == EOPNOTSUPP || - errno == ENXIO) { + errno == ENOENT || errno == ENXIO) { if (verbose > 0) ui__warning("%s event is not supported by the kernel.\n", evsel__name(counter)); @@ -658,7 +658,7 @@ static enum counter_recovery stat_handle_error(struct evsel *counter) if (verbose > 0) ui__warning("%s\n", msg); return COUNTER_RETRY; - } else if (target__has_per_thread(&target) && + } else if (target__has_per_thread(&target) && errno != EOPNOTSUPP && evsel_list->core.threads && evsel_list->core.threads->err_thread != -1) { /* @@ -679,6 +679,19 @@ static enum counter_recovery stat_handle_error(struct evsel *counter) return COUNTER_SKIP; } + if (errno == EOPNOTSUPP) { + if (verbose > 0) { + ui__warning("%s event is not supported by the kernel.\n", + evsel__name(counter)); + } + counter->supported = false; + counter->errored = true; + + if ((evsel__leader(counter) != counter) || + !(counter->core.leader->nr_members > 1)) + return COUNTER_SKIP; + } + evsel__open_strerror(counter, &target, errno, msg, sizeof(msg)); ui__error("%s\n", msg); @@ -716,15 +729,19 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) } if (!cpu_map__is_dummy(evsel_list->core.user_requested_cpus)) { - if (affinity__setup(&saved_affinity) < 0) - return -1; + if (affinity__setup(&saved_affinity) < 0) { + err = -1; + goto err_out; + } affinity = &saved_affinity; } evlist__for_each_entry(evsel_list, counter) { counter->reset_group = false; - if (bpf_counter__load(counter, &target)) - return -1; + if (bpf_counter__load(counter, &target)) { + err = -1; + goto err_out; + } if (!(evsel__is_bperf(counter))) all_counters_use_bpf = false; } @@ -767,7 +784,8 @@ try_again: switch (stat_handle_error(counter)) { case COUNTER_FATAL: - return -1; + err = -1; + goto err_out; case COUNTER_RETRY: goto try_again; case COUNTER_SKIP: @@ -808,7 +826,8 @@ try_again_reset: switch (stat_handle_error(counter)) { case COUNTER_FATAL: - return -1; + err = -1; + goto err_out; case COUNTER_RETRY: goto try_again_reset; case COUNTER_SKIP: @@ -821,6 +840,7 @@ try_again_reset: } } affinity__cleanup(affinity); + affinity = NULL; evlist__for_each_entry(evsel_list, counter) { if (!counter->supported) { @@ -833,8 +853,10 @@ try_again_reset: stat_config.unit_width = l; if (evsel__should_store_id(counter) && - evsel__store_ids(counter, evsel_list)) - return -1; + evsel__store_ids(counter, evsel_list)) { + err = -1; + goto err_out; + } } if (evlist__apply_filters(evsel_list, &counter, &target)) { @@ -855,20 +877,23 @@ try_again_reset: } if (err < 0) - return err; + goto err_out; err = perf_event__synthesize_stat_events(&stat_config, NULL, evsel_list, process_synthesized_event, is_pipe); if (err < 0) - return err; + goto err_out; + } if (target.initial_delay) { pr_info(EVLIST_DISABLED_MSG); } else { err = enable_counters(); - if (err) - return -1; + if (err) { + err = -1; + goto err_out; + } } /* Exec the command, if any */ @@ -878,8 +903,10 @@ try_again_reset: if (target.initial_delay > 0) { usleep(target.initial_delay * USEC_PER_MSEC); err = enable_counters(); - if (err) - return -1; + if (err) { + err = -1; + goto err_out; + } pr_info(EVLIST_ENABLED_MSG); } @@ -899,7 +926,8 @@ try_again_reset: if (workload_exec_errno) { const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg)); pr_err("Workload failed: %s\n", emsg); - return -1; + err = -1; + goto err_out; } if (WIFSIGNALED(status)) @@ -946,8 +974,23 @@ try_again_reset: evlist__close(evsel_list); return WEXITSTATUS(status); + +err_out: + if (forks) + evlist__cancel_workload(evsel_list); + + affinity__cleanup(affinity); + return err; } +/* + * Returns -1 for fatal errors which signifies to not continue + * when in repeat mode. + * + * Returns < -1 error codes when stat record is used. These + * result in the stat information being displayed, but writing + * to the file fails and is non fatal. + */ static int run_perf_stat(int argc, const char **argv, int run_idx) { int ret; @@ -1814,130 +1857,25 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) } /* - * Add default attributes, if there were no attributes specified or + * Add default events, if there were no attributes specified or * if -d/--detailed, -d -d or -d -d -d is used: */ -static int add_default_attributes(void) +static int add_default_events(void) { - struct perf_event_attr default_attrs0[] = { - - { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_TASK_CLOCK }, - { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_CONTEXT_SWITCHES }, - { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_CPU_MIGRATIONS }, - { .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_PAGE_FAULTS }, - - { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES }, -}; - struct perf_event_attr frontend_attrs[] = { - { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND }, -}; - struct perf_event_attr backend_attrs[] = { - { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_BACKEND }, -}; - struct perf_event_attr default_attrs1[] = { - { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS }, - { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, - { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_MISSES }, - -}; - -/* - * Detailed stats (-d), covering the L1 and last level data caches: - */ - struct perf_event_attr detailed_attrs[] = { - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_L1D << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_L1D << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_LL << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_LL << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, -}; - -/* - * Very detailed stats (-d -d), covering the instruction cache and the TLB caches: - */ - struct perf_event_attr very_detailed_attrs[] = { - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_L1I << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_L1I << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_DTLB << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_DTLB << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_ITLB << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_ITLB << 0 | - (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, - -}; + const char *pmu = parse_events_option_args.pmu_filter ?: "all"; + struct parse_events_error err; + struct evlist *evlist = evlist__new(); + struct evsel *evsel; + int ret = 0; -/* - * Very, very detailed stats (-d -d -d), adding prefetch events: - */ - struct perf_event_attr very_very_detailed_attrs[] = { - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_L1D << 0 | - (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | - (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) }, - - { .type = PERF_TYPE_HW_CACHE, - .config = - PERF_COUNT_HW_CACHE_L1D << 0 | - (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) }, -}; + if (!evlist) + return -ENOMEM; - struct perf_event_attr default_null_attrs[] = {}; - const char *pmu = parse_events_option_args.pmu_filter ?: "all"; + parse_events_error__init(&err); /* Set attrs if no event is selected and !null_run: */ if (stat_config.null_run) - return 0; + goto out; if (transaction_run) { /* Handle -T as -M transaction. Once platform specific metrics @@ -1947,9 +1885,10 @@ static int add_default_attributes(void) */ if (!metricgroup__has_metric(pmu, "transaction")) { pr_err("Missing transaction metrics\n"); - return -1; + ret = -1; + goto out; } - return metricgroup__parse_groups(evsel_list, pmu, "transaction", + ret = metricgroup__parse_groups(evlist, pmu, "transaction", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -1957,6 +1896,7 @@ static int add_default_attributes(void) stat_config.system_wide, stat_config.hardware_aware_grouping, &stat_config.metric_events); + goto out; } if (smi_cost) { @@ -1964,26 +1904,29 @@ static int add_default_attributes(void) if (sysfs__read_int(FREEZE_ON_SMI_PATH, &smi) < 0) { pr_err("freeze_on_smi is not supported.\n"); - return -1; + ret = -1; + goto out; } if (!smi) { if (sysfs__write_int(FREEZE_ON_SMI_PATH, 1) < 0) { - fprintf(stderr, "Failed to set freeze_on_smi.\n"); - return -1; + pr_err("Failed to set freeze_on_smi.\n"); + ret = -1; + goto out; } smi_reset = true; } if (!metricgroup__has_metric(pmu, "smi")) { pr_err("Missing smi metrics\n"); - return -1; + ret = -1; + goto out; } if (!force_metric_only) stat_config.metric_only = true; - return metricgroup__parse_groups(evsel_list, pmu, "smi", + ret = metricgroup__parse_groups(evlist, pmu, "smi", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -1991,6 +1934,7 @@ static int add_default_attributes(void) stat_config.system_wide, stat_config.hardware_aware_grouping, &stat_config.metric_events); + goto out; } if (topdown_run) { @@ -2003,21 +1947,23 @@ static int add_default_attributes(void) if (!max_level) { pr_err("Topdown requested but the topdown metric groups aren't present.\n" "(See perf list the metric groups have names like TopdownL1)\n"); - return -1; + ret = -1; + goto out; } if (stat_config.topdown_level > max_level) { pr_err("Invalid top-down metrics level. The max level is %u.\n", max_level); - return -1; - } else if (!stat_config.topdown_level) + ret = -1; + goto out; + } else if (!stat_config.topdown_level) { stat_config.topdown_level = 1; - + } if (!stat_config.interval && !stat_config.metric_only) { fprintf(stat_config.output, "Topdown accuracy may decrease when measuring long periods.\n" "Please print the result regularly, e.g. -I1000\n"); } str[8] = stat_config.topdown_level + '0'; - if (metricgroup__parse_groups(evsel_list, + if (metricgroup__parse_groups(evlist, pmu, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, @@ -2025,41 +1971,49 @@ static int add_default_attributes(void) stat_config.user_requested_cpu_list, stat_config.system_wide, stat_config.hardware_aware_grouping, - &stat_config.metric_events) < 0) - return -1; + &stat_config.metric_events) < 0) { + ret = -1; + goto out; + } } if (!stat_config.topdown_level) stat_config.topdown_level = 1; - if (!evsel_list->core.nr_entries) { + if (!evlist->core.nr_entries && !evsel_list->core.nr_entries) { /* No events so add defaults. */ if (target__has_cpu(&target)) - default_attrs0[0].config = PERF_COUNT_SW_CPU_CLOCK; + ret = parse_events(evlist, "cpu-clock", &err); + else + ret = parse_events(evlist, "task-clock", &err); + if (ret) + goto out; + + ret = parse_events(evlist, + "context-switches," + "cpu-migrations," + "page-faults," + "instructions," + "cycles," + "stalled-cycles-frontend," + "stalled-cycles-backend," + "branches," + "branch-misses", + &err); + if (ret) + goto out; - if (evlist__add_default_attrs(evsel_list, default_attrs0) < 0) - return -1; - if (perf_pmus__have_event("cpu", "stalled-cycles-frontend")) { - if (evlist__add_default_attrs(evsel_list, frontend_attrs) < 0) - return -1; - } - if (perf_pmus__have_event("cpu", "stalled-cycles-backend")) { - if (evlist__add_default_attrs(evsel_list, backend_attrs) < 0) - return -1; - } - if (evlist__add_default_attrs(evsel_list, default_attrs1) < 0) - return -1; /* * Add TopdownL1 metrics if they exist. To minimize * multiplexing, don't request threshold computation. */ if (metricgroup__has_metric(pmu, "Default")) { struct evlist *metric_evlist = evlist__new(); - struct evsel *metric_evsel; - - if (!metric_evlist) - return -1; + if (!metric_evlist) { + ret = -ENOMEM; + goto out; + } if (metricgroup__parse_groups(metric_evlist, pmu, "Default", /*metric_no_group=*/false, /*metric_no_merge=*/false, @@ -2067,43 +2021,71 @@ static int add_default_attributes(void) stat_config.user_requested_cpu_list, stat_config.system_wide, stat_config.hardware_aware_grouping, - &stat_config.metric_events) < 0) - return -1; - - evlist__for_each_entry(metric_evlist, metric_evsel) { - metric_evsel->skippable = true; - metric_evsel->default_metricgroup = true; + &stat_config.metric_events) < 0) { + ret = -1; + goto out; } - evlist__splice_list_tail(evsel_list, &metric_evlist->core.entries); + + evlist__for_each_entry(metric_evlist, evsel) + evsel->default_metricgroup = true; + + evlist__splice_list_tail(evlist, &metric_evlist->core.entries); evlist__delete(metric_evlist); } - - /* Platform specific attrs */ - if (evlist__add_default_attrs(evsel_list, default_null_attrs) < 0) - return -1; } /* Detailed events get appended to the event list: */ - if (detailed_run < 1) - return 0; - - /* Append detailed run extra attributes: */ - if (evlist__add_default_attrs(evsel_list, detailed_attrs) < 0) - return -1; - - if (detailed_run < 2) - return 0; - - /* Append very detailed run extra attributes: */ - if (evlist__add_default_attrs(evsel_list, very_detailed_attrs) < 0) - return -1; - - if (detailed_run < 3) - return 0; - - /* Append very, very detailed run extra attributes: */ - return evlist__add_default_attrs(evsel_list, very_very_detailed_attrs); + if (!ret && detailed_run >= 1) { + /* + * Detailed stats (-d), covering the L1 and last level data + * caches: + */ + ret = parse_events(evlist, + "L1-dcache-loads," + "L1-dcache-load-misses," + "LLC-loads," + "LLC-load-misses", + &err); + } + if (!ret && detailed_run >= 2) { + /* + * Very detailed stats (-d -d), covering the instruction cache + * and the TLB caches: + */ + ret = parse_events(evlist, + "L1-icache-loads," + "L1-icache-load-misses," + "dTLB-loads," + "dTLB-load-misses," + "iTLB-loads," + "iTLB-load-misses", + &err); + } + if (!ret && detailed_run >= 3) { + /* + * Very, very detailed stats (-d -d -d), adding prefetch events: + */ + ret = parse_events(evlist, + "L1-dcache-prefetches," + "L1-dcache-prefetch-misses", + &err); + } +out: + if (!ret) { + evlist__for_each_entry(evlist, evsel) { + /* + * Make at least one event non-skippable so fatal errors are visible. + * 'cycles' always used to be default and non-skippable, so use that. + */ + if (strcmp("cycles", evsel__name(evsel))) + evsel->skippable = true; + } + } + parse_events_error__exit(&err); + evlist__splice_list_tail(evsel_list, &evlist->core.entries); + evlist__delete(evlist); + return ret; } static const char * const stat_record_usage[] = { @@ -2591,6 +2573,14 @@ int cmd_stat(int argc, const char **argv) goto out; } + if (stat_config.csv_output || (stat_config.metric_only && stat_config.json_output)) { + /* + * Current CSV and metric-only JSON output doesn't display the + * metric threshold so don't compute it. + */ + stat_config.metric_no_threshold = true; + } + if (stat_config.walltime_run_table && stat_config.run_count <= 1) { fprintf(stderr, "--table is only supported with -r\n"); parse_options_usage(stat_usage, stat_options, "r", 1); @@ -2651,6 +2641,7 @@ int cmd_stat(int argc, const char **argv) } else if (big_num_opt == 0) /* User passed --no-big-num */ stat_config.big_num = false; + target.inherit = !stat_config.no_inherit; err = target__validate(&target); if (err) { target__strerror(&target, err, errbuf, BUFSIZ); @@ -2760,7 +2751,7 @@ int cmd_stat(int argc, const char **argv) } } - if (add_default_attributes()) + if (add_default_events()) goto out; if (stat_config.cgroup_list) { @@ -2879,7 +2870,10 @@ int cmd_stat(int argc, const char **argv) evlist__reset_prev_raw_counts(evsel_list); status = run_perf_stat(argc, argv, run_idx); - if (forever && status != -1 && !interval) { + if (status == -1) + break; + + if (forever && !interval) { print_counters(NULL, argc, argv); perf_stat__reset_stats(); } diff --git a/tools/perf/builtin-timechart.c b/tools/perf/builtin-timechart.c index 218c8b44d7be..068d297aaf44 100644 --- a/tools/perf/builtin-timechart.c +++ b/tools/perf/builtin-timechart.c @@ -38,7 +38,7 @@ #include "util/tracepoint.h" #include "util/util.h" #include <linux/err.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> #ifdef LACKS_OPEN_MEMSTREAM_PROTOTYPE FILE *open_memstream(char **ptr, size_t *sizeloc); @@ -1158,7 +1158,6 @@ static void draw_io_bars(struct timechart *tchart) } svg_box(Y, c->start_time, c->end_time, "process3"); - sample = c->io_samples; for (sample = c->io_samples; sample; sample = sample->next) { double h = (double)sample->bytes / c->max_bytes; diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index d3f11b90d025..6a1a128fe645 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -88,7 +88,7 @@ #include <perf/mmap.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif #ifndef O_CLOEXEC @@ -1873,7 +1873,7 @@ static int trace__process_event(struct trace *trace, struct machine *machine, switch (event->header.type) { case PERF_RECORD_LOST: color_fprintf(trace->output, PERF_COLOR_RED, - "LOST %" PRIu64 " events!\n", event->lost.lost); + "LOST %" PRIu64 " events!\n", (u64)event->lost.lost); ret = machine__process_lost_event(machine, event, sample); break; default: @@ -2702,6 +2702,7 @@ static int trace__fprintf_sys_enter(struct trace *trace, struct evsel *evsel, char msg[1024]; void *args, *augmented_args = NULL; int augmented_args_size; + size_t printed = 0; if (sc == NULL) return -1; @@ -2717,8 +2718,8 @@ static int trace__fprintf_sys_enter(struct trace *trace, struct evsel *evsel, args = perf_evsel__sc_tp_ptr(evsel, args, sample); augmented_args = syscall__augmented_args(sc, sample, &augmented_args_size, trace->raw_augmented_syscalls_args_size); - syscall__scnprintf_args(sc, msg, sizeof(msg), args, augmented_args, augmented_args_size, trace, thread); - fprintf(trace->output, "%s", msg); + printed += syscall__scnprintf_args(sc, msg, sizeof(msg), args, augmented_args, augmented_args_size, trace, thread); + fprintf(trace->output, "%.*s", (int)printed, msg); err = 0; out_put: thread__put(thread); @@ -3087,7 +3088,7 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct evsel *evsel, printed += syscall_arg_fmt__scnprintf_val(arg, bf + printed, size - printed, &syscall_arg, val); } - return printed + fprintf(trace->output, "%s", bf); + return printed + fprintf(trace->output, "%.*s", (int)printed, bf); } static int trace__event_handler(struct trace *trace, struct evsel *evsel, @@ -3096,13 +3097,8 @@ static int trace__event_handler(struct trace *trace, struct evsel *evsel, { struct thread *thread; int callchain_ret = 0; - /* - * Check if we called perf_evsel__disable(evsel) due to, for instance, - * this event's max_events having been hit and this is an entry coming - * from the ring buffer that we should discard, since the max events - * have already been considered/printed. - */ - if (evsel->disabled) + + if (evsel->nr_events_printed >= evsel->max_events) return 0; thread = machine__findnew_thread(trace->host, sample->pid, sample->tid); @@ -4326,6 +4322,9 @@ static int trace__run(struct trace *trace, int argc, const char **argv) sizeof(__u32), BPF_ANY); } } + + if (trace->skel) + trace->filter_pids.map = trace->skel->maps.pids_filtered; #endif err = trace__set_filter_pids(trace); if (err < 0) @@ -5449,6 +5448,10 @@ init_augmented_syscall_tp: if (trace.summary_only) trace.summary = trace.summary_only; + /* Keep exited threads, otherwise information might be lost for summary */ + if (trace.summary) + symbol_conf.keep_exited_threads = true; + if (output_name != NULL) { err = trace__open_output(&trace, output_name); if (err < 0) { diff --git a/tools/perf/dlfilters/dlfilter-test-api-v0.c b/tools/perf/dlfilters/dlfilter-test-api-v0.c index 4083b1abeaab..4ca2d7b2ea6c 100644 --- a/tools/perf/dlfilters/dlfilter-test-api-v0.c +++ b/tools/perf/dlfilters/dlfilter-test-api-v0.c @@ -220,7 +220,7 @@ static int check_sample(struct filter_data *d, const struct perf_dlfilter_sample CHECK_SAMPLE(raw_callchain_nr); CHECK(!sample->raw_callchain); -#define EVENT_NAME "branches:" +#define EVENT_NAME "branches" CHECK(!strncmp(sample->event, EVENT_NAME, strlen(EVENT_NAME))); return 0; diff --git a/tools/perf/dlfilters/dlfilter-test-api-v2.c b/tools/perf/dlfilters/dlfilter-test-api-v2.c index 32ff619e881c..00d73a16c4fd 100644 --- a/tools/perf/dlfilters/dlfilter-test-api-v2.c +++ b/tools/perf/dlfilters/dlfilter-test-api-v2.c @@ -235,7 +235,7 @@ static int check_sample(struct filter_data *d, const struct perf_dlfilter_sample CHECK_SAMPLE(raw_callchain_nr); CHECK(!sample->raw_callchain); -#define EVENT_NAME "branches:" +#define EVENT_NAME "branches" CHECK(!strncmp(sample->event, EVENT_NAME, strlen(EVENT_NAME))); return 0; diff --git a/tools/perf/perf.c b/tools/perf/perf.c index 4def800f4089..a2987f2cfe1a 100644 --- a/tools/perf/perf.c +++ b/tools/perf/perf.c @@ -542,8 +542,6 @@ int main(int argc, const char **argv) } cmd = argv[0]; - test_attr__init(); - /* * We use PATH to find perf commands, but we prepend some higher * precedence paths: the "--exec-path" option, the PERF_EXEC_PATH diff --git a/tools/perf/pmu-events/arch/arm64/freescale/imx91/sys/ddrc.json b/tools/perf/pmu-events/arch/arm64/freescale/imx91/sys/ddrc.json new file mode 100644 index 000000000000..74ac12660a29 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/freescale/imx91/sys/ddrc.json @@ -0,0 +1,9 @@ +[ + { + "BriefDescription": "ddr cycles event", + "EventCode": "0x00", + "EventName": "imx91_ddr.cycles", + "Unit": "imx9_ddr", + "Compat": "imx91" + } +] diff --git a/tools/perf/pmu-events/arch/arm64/freescale/imx91/sys/metrics.json b/tools/perf/pmu-events/arch/arm64/freescale/imx91/sys/metrics.json new file mode 100644 index 000000000000..f0c5911eb2d0 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/freescale/imx91/sys/metrics.json @@ -0,0 +1,26 @@ +[ + { + "BriefDescription": "bandwidth usage for lpddr4 evk board", + "MetricName": "imx91_bandwidth_usage.lpddr4", + "MetricExpr": "(((( imx9_ddr0@ddrc_pm_0@ ) * 2 * 8 ) + (( imx9_ddr0@ddrc_pm_3@ + imx9_ddr0@ddrc_pm_5@ + imx9_ddr0@ddrc_pm_7@ + imx9_ddr0@ddrc_pm_9@ - imx9_ddr0@ddrc_pm_2@ - imx9_ddr0@ddrc_pm_4@ - imx9_ddr0@ddrc_pm_6@ - imx9_ddr0@ddrc_pm_8@ ) * 32 )) / duration_time) / (2400 * 1000000 * 2)", + "ScaleUnit": "1e2%", + "Unit": "imx9_ddr", + "Compat": "imx91" + }, + { + "BriefDescription": "bytes all masters read from ddr", + "MetricName": "imx91_ddr_read.all", + "MetricExpr": "( imx9_ddr0@ddrc_pm_0@ ) * 2 * 8", + "ScaleUnit": "9.765625e-4KB", + "Unit": "imx9_ddr", + "Compat": "imx91" + }, + { + "BriefDescription": "bytes all masters write to ddr", + "MetricName": "imx91_ddr_write.all", + "MetricExpr": "( imx9_ddr0@ddrc_pm_3@ + imx9_ddr0@ddrc_pm_5@ + imx9_ddr0@ddrc_pm_7@ + imx9_ddr0@ddrc_pm_9@ - imx9_ddr0@ddrc_pm_2@ - imx9_ddr0@ddrc_pm_4@ - imx9_ddr0@ddrc_pm_6@ - imx9_ddr0@ddrc_pm_8@ ) * 32", + "ScaleUnit": "9.765625e-4KB", + "Unit": "imx9_ddr", + "Compat": "imx91" + } +] diff --git a/tools/perf/pmu-events/arch/arm64/freescale/imx95/sys/metrics.json b/tools/perf/pmu-events/arch/arm64/freescale/imx95/sys/metrics.json index 126ce980f6f2..45a0d51dfb63 100644 --- a/tools/perf/pmu-events/arch/arm64/freescale/imx95/sys/metrics.json +++ b/tools/perf/pmu-events/arch/arm64/freescale/imx95/sys/metrics.json @@ -8,6 +8,14 @@ "Compat": "imx95" }, { + "BriefDescription": "bandwidth usage for lpddr4x evk board", + "MetricName": "imx95_bandwidth_usage.lpddr4x", + "MetricExpr": "(( imx9_ddr0@eddrtq_pm_rd_beat_filt0\\,axi_mask\\=0x000\\,axi_id\\=0x000@ + imx9_ddr0@eddrtq_pm_wr_beat_filt\\,axi_mask\\=0x000\\,axi_id\\=0x000@ ) * 32 / duration_time) / (4000 * 1000000 * 4)", + "ScaleUnit": "1e2%", + "Unit": "imx9_ddr", + "Compat": "imx95" + }, + { "BriefDescription": "bytes of all masters read from ddr", "MetricName": "imx95_ddr_read.all", "MetricExpr": "( imx9_ddr0@eddrtq_pm_rd_beat_filt0\\,axi_mask\\=0x000\\,axi_id\\=0x000@ ) * 32", diff --git a/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json b/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json index 6463531b9941..b6a0d2de8534 100644 --- a/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json +++ b/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json @@ -3,235 +3,235 @@ "MetricExpr": "FETCH_BUBBLE / (4 * CPU_CYCLES)", "PublicDescription": "Frontend bound L1 topdown metric", "BriefDescription": "Frontend bound L1 topdown metric", - "DefaultMetricgroupName": "TopDownL1", - "MetricGroup": "Default;TopDownL1", + "DefaultMetricgroupName": "TopdownL1", + "MetricGroup": "Default;TopdownL1", "MetricName": "frontend_bound" }, { "MetricExpr": "(INST_SPEC - INST_RETIRED) / (4 * CPU_CYCLES)", "PublicDescription": "Bad Speculation L1 topdown metric", "BriefDescription": "Bad Speculation L1 topdown metric", - "DefaultMetricgroupName": "TopDownL1", - "MetricGroup": "Default;TopDownL1", + "DefaultMetricgroupName": "TopdownL1", + "MetricGroup": "Default;TopdownL1", "MetricName": "bad_speculation" }, { "MetricExpr": "INST_RETIRED / (CPU_CYCLES * 4)", "PublicDescription": "Retiring L1 topdown metric", "BriefDescription": "Retiring L1 topdown metric", - "DefaultMetricgroupName": "TopDownL1", - "MetricGroup": "Default;TopDownL1", + "DefaultMetricgroupName": "TopdownL1", + "MetricGroup": "Default;TopdownL1", "MetricName": "retiring" }, { "MetricExpr": "1 - (frontend_bound + bad_speculation + retiring)", "PublicDescription": "Backend Bound L1 topdown metric", "BriefDescription": "Backend Bound L1 topdown metric", - "DefaultMetricgroupName": "TopDownL1", - "MetricGroup": "Default;TopDownL1", + "DefaultMetricgroupName": "TopdownL1", + "MetricGroup": "Default;TopdownL1", "MetricName": "backend_bound" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x201d@ / CPU_CYCLES", "PublicDescription": "Fetch latency bound L2 topdown metric", "BriefDescription": "Fetch latency bound L2 topdown metric", - "MetricGroup": "TopDownL2", + "MetricGroup": "TopdownL2", "MetricName": "fetch_latency_bound" }, { "MetricExpr": "frontend_bound - fetch_latency_bound", "PublicDescription": "Fetch bandwidth bound L2 topdown metric", "BriefDescription": "Fetch bandwidth bound L2 topdown metric", - "MetricGroup": "TopDownL2", + "MetricGroup": "TopdownL2", "MetricName": "fetch_bandwidth_bound" }, { "MetricExpr": "(bad_speculation * BR_MIS_PRED) / (BR_MIS_PRED + armv8_pmuv3_0@event\\=0x2013@)", "PublicDescription": "Branch mispredicts L2 topdown metric", "BriefDescription": "Branch mispredicts L2 topdown metric", - "MetricGroup": "TopDownL2", + "MetricGroup": "TopdownL2", "MetricName": "branch_mispredicts" }, { "MetricExpr": "bad_speculation - branch_mispredicts", "PublicDescription": "Machine clears L2 topdown metric", "BriefDescription": "Machine clears L2 topdown metric", - "MetricGroup": "TopDownL2", + "MetricGroup": "TopdownL2", "MetricName": "machine_clears" }, { "MetricExpr": "(EXE_STALL_CYCLE - (MEM_STALL_ANYLOAD + armv8_pmuv3_0@event\\=0x7005@)) / CPU_CYCLES", "PublicDescription": "Core bound L2 topdown metric", "BriefDescription": "Core bound L2 topdown metric", - "MetricGroup": "TopDownL2", + "MetricGroup": "TopdownL2", "MetricName": "core_bound" }, { "MetricExpr": "(MEM_STALL_ANYLOAD + armv8_pmuv3_0@event\\=0x7005@) / CPU_CYCLES", "PublicDescription": "Memory bound L2 topdown metric", "BriefDescription": "Memory bound L2 topdown metric", - "MetricGroup": "TopDownL2", + "MetricGroup": "TopdownL2", "MetricName": "memory_bound" }, { "MetricExpr": "(((L2I_TLB - L2I_TLB_REFILL) * 15) + (L2I_TLB_REFILL * 100)) / CPU_CYCLES", "PublicDescription": "Idle by itlb miss L3 topdown metric", "BriefDescription": "Idle by itlb miss L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "idle_by_itlb_miss" }, { "MetricExpr": "(((L2I_CACHE - L2I_CACHE_REFILL) * 15) + (L2I_CACHE_REFILL * 100)) / CPU_CYCLES", "PublicDescription": "Idle by icache miss L3 topdown metric", "BriefDescription": "Idle by icache miss L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "idle_by_icache_miss" }, { "MetricExpr": "(BR_MIS_PRED * 5) / CPU_CYCLES", "PublicDescription": "BP misp flush L3 topdown metric", "BriefDescription": "BP misp flush L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "bp_misp_flush" }, { "MetricExpr": "(armv8_pmuv3_0@event\\=0x2013@ * 5) / CPU_CYCLES", "PublicDescription": "OOO flush L3 topdown metric", "BriefDescription": "OOO flush L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "ooo_flush" }, { "MetricExpr": "(armv8_pmuv3_0@event\\=0x1001@ * 5) / CPU_CYCLES", "PublicDescription": "Static predictor flush L3 topdown metric", "BriefDescription": "Static predictor flush L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "sp_flush" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x1010@ / BR_MIS_PRED", "PublicDescription": "Indirect branch L3 topdown metric", "BriefDescription": "Indirect branch L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "indirect_branch" }, { "MetricExpr": "(armv8_pmuv3_0@event\\=0x1013@ + armv8_pmuv3_0@event\\=0x1016@) / BR_MIS_PRED", "PublicDescription": "Push branch L3 topdown metric", "BriefDescription": "Push branch L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "push_branch" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x100d@ / BR_MIS_PRED", "PublicDescription": "Pop branch L3 topdown metric", "BriefDescription": "Pop branch L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "pop_branch" }, { "MetricExpr": "(BR_MIS_PRED - armv8_pmuv3_0@event\\=0x1010@ - armv8_pmuv3_0@event\\=0x1013@ - armv8_pmuv3_0@event\\=0x1016@ - armv8_pmuv3_0@event\\=0x100d@) / BR_MIS_PRED", "PublicDescription": "Other branch L3 topdown metric", "BriefDescription": "Other branch L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "other_branch" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x2012@ / armv8_pmuv3_0@event\\=0x2013@", "PublicDescription": "Nuke flush L3 topdown metric", "BriefDescription": "Nuke flush L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "nuke_flush" }, { "MetricExpr": "1 - nuke_flush", "PublicDescription": "Other flush L3 topdown metric", "BriefDescription": "Other flush L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "other_flush" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x2010@ / CPU_CYCLES", "PublicDescription": "Sync stall L3 topdown metric", "BriefDescription": "Sync stall L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "sync_stall" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x2004@ / CPU_CYCLES", "PublicDescription": "Rob stall L3 topdown metric", "BriefDescription": "Rob stall L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "rob_stall" }, { "MetricExpr": "(armv8_pmuv3_0@event\\=0x2006@ + armv8_pmuv3_0@event\\=0x2007@ + armv8_pmuv3_0@event\\=0x2008@) / CPU_CYCLES", "PublicDescription": "Ptag stall L3 topdown metric", "BriefDescription": "Ptag stall L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "ptag_stall" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x201e@ / CPU_CYCLES", "PublicDescription": "SaveOpQ stall L3 topdown metric", "BriefDescription": "SaveOpQ stall L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "saveopq_stall" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x2005@ / CPU_CYCLES", "PublicDescription": "PC buffer stall L3 topdown metric", "BriefDescription": "PC buffer stall L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "pc_buffer_stall" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x7002@ / CPU_CYCLES", "PublicDescription": "Divider L3 topdown metric", "BriefDescription": "Divider L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "divider" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x7003@ / CPU_CYCLES", "PublicDescription": "FSU stall L3 topdown metric", "BriefDescription": "FSU stall L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "fsu_stall" }, { "MetricExpr": "core_bound - divider - fsu_stall", "PublicDescription": "EXE ports util L3 topdown metric", "BriefDescription": "EXE ports util L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "exe_ports_util" }, { "MetricExpr": "(MEM_STALL_ANYLOAD - MEM_STALL_L1MISS) / CPU_CYCLES", "PublicDescription": "L1 bound L3 topdown metric", "BriefDescription": "L1 bound L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "l1_bound" }, { "MetricExpr": "(MEM_STALL_L1MISS - MEM_STALL_L2MISS) / CPU_CYCLES", "PublicDescription": "L2 bound L3 topdown metric", "BriefDescription": "L2 bound L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "l2_bound" }, { "MetricExpr": "MEM_STALL_L2MISS / CPU_CYCLES", "PublicDescription": "Mem bound L3 topdown metric", "BriefDescription": "Mem bound L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "mem_bound" }, { "MetricExpr": "armv8_pmuv3_0@event\\=0x7005@ / CPU_CYCLES", "PublicDescription": "Store bound L3 topdown metric", "BriefDescription": "Store bound L3 topdown metric", - "MetricGroup": "TopDownL3", + "MetricGroup": "TopdownL3", "MetricName": "store_bound" } ] diff --git a/tools/perf/pmu-events/arch/common/common/tool.json b/tools/perf/pmu-events/arch/common/common/tool.json new file mode 100644 index 000000000000..12f2ef1813a6 --- /dev/null +++ b/tools/perf/pmu-events/arch/common/common/tool.json @@ -0,0 +1,74 @@ +[ + { + "Unit": "tool", + "EventName": "duration_time", + "BriefDescription": "Wall clock interval time in nanoseconds", + "ConfigCode": "1" + }, + { + "Unit": "tool", + "EventName": "user_time", + "BriefDescription": "User (non-kernel) time in nanoseconds", + "ConfigCode": "2" + }, + { + "Unit": "tool", + "EventName": "system_time", + "BriefDescription": "System/kernel time in nanoseconds", + "ConfigCode": "3" + }, + { + "Unit": "tool", + "EventName": "has_pmem", + "BriefDescription": "1 if persistent memory installed otherwise 0", + "ConfigCode": "4" + }, + { + "Unit": "tool", + "EventName": "num_cores", + "BriefDescription": "Number of cores. A core consists of 1 or more thread, with each thread being associated with a logical Linux CPU", + "ConfigCode": "5" + }, + { + "Unit": "tool", + "EventName": "num_cpus", + "BriefDescription": "Number of logical Linux CPUs. There may be multiple such CPUs on a core", + "ConfigCode": "6" + }, + { + "Unit": "tool", + "EventName": "num_cpus_online", + "BriefDescription": "Number of online logical Linux CPUs. There may be multiple such CPUs on a core", + "ConfigCode": "7" + }, + { + "Unit": "tool", + "EventName": "num_dies", + "BriefDescription": "Number of dies. Each die has 1 or more cores", + "ConfigCode": "8" + }, + { + "Unit": "tool", + "EventName": "num_packages", + "BriefDescription": "Number of packages. Each package has 1 or more die", + "ConfigCode": "9" + }, + { + "Unit": "tool", + "EventName": "slots", + "BriefDescription": "Number of functional units that in parallel can execute parts of an instruction", + "ConfigCode": "10" + }, + { + "Unit": "tool", + "EventName": "smt_on", + "BriefDescription": "1 if simultaneous multithreading (aka hyperthreading) is enable otherwise 0", + "ConfigCode": "11" + }, + { + "Unit": "tool", + "EventName": "system_tsc_freq", + "BriefDescription": "The amount a Time Stamp Counter (TSC) increases per second", + "ConfigCode": "12" + } +] diff --git a/tools/perf/pmu-events/arch/powerpc/compat/generic-events.json b/tools/perf/pmu-events/arch/powerpc/compat/generic-events.json new file mode 100644 index 000000000000..6f5e8efcb098 --- /dev/null +++ b/tools/perf/pmu-events/arch/powerpc/compat/generic-events.json @@ -0,0 +1,117 @@ +[ + { + "EventCode": "0x600F4", + "EventName": "PM_CYC", + "BriefDescription": "Processor cycles." + }, + { + "EventCode": "0x100F2", + "EventName": "PM_CYC_INST_CMPL", + "BriefDescription": "1 or more ppc insts finished" + }, + { + "EventCode": "0x100f4", + "EventName": "PM_FLOP_CMPL", + "BriefDescription": "Floating Point Operations Finished." + }, + { + "EventCode": "0x100F6", + "EventName": "PM_L1_ITLB_MISS", + "BriefDescription": "Number of I-ERAT reloads." + }, + { + "EventCode": "0x100F8", + "EventName": "PM_NO_INST_AVAIL", + "BriefDescription": "Number of cycles the ICT has no itags assigned to this thread." + }, + { + "EventCode": "0x100fc", + "EventName": "PM_LD_CMPL", + "BriefDescription": "Load instruction completed." + }, + { + "EventCode": "0x200F0", + "EventName": "PM_ST_CMPL", + "BriefDescription": "Stores completed from S2Q (2nd-level store queue)." + }, + { + "EventCode": "0x200F2", + "EventName": "PM_INST_DISP", + "BriefDescription": "PowerPC instruction dispatched." + }, + { + "EventCode": "0x200F4", + "EventName": "PM_RUN_CYC", + "BriefDescription": "Processor cycles gated by the run latch." + }, + { + "EventCode": "0x200F6", + "EventName": "PM_L1_DTLB_RELOAD", + "BriefDescription": "DERAT Reloaded due to a DERAT miss." + }, + { + "EventCode": "0x200FA", + "EventName": "PM_BR_TAKEN_CMPL", + "BriefDescription": "Branch Taken instruction completed." + }, + { + "EventCode": "0x200FC", + "EventName": "PM_L1_ICACHE_MISS", + "BriefDescription": "Demand instruction cache miss." + }, + { + "EventCode": "0x200FE", + "EventName": "PM_L1_RELOAD_FROM_MEM", + "BriefDescription": "L1 Dcache reload from memory" + }, + { + "EventCode": "0x300F0", + "EventName": "PM_ST_MISS_L1", + "BriefDescription": "Store Missed L1" + }, + { + "EventCode": "0x300FC", + "EventName": "PM_DTLB_MISS", + "BriefDescription": "Data PTEG reload" + }, + { + "EventCode": "0x300FE", + "EventName": "PM_DATA_FROM_L3MISS", + "BriefDescription": "Demand LD - L3 Miss (not L2 hit and not L3 hit)" + }, + { + "EventCode": "0x400F0", + "EventName": "PM_LD_MISS_L1", + "BriefDescription": "L1 Dcache load miss" + }, + { + "EventCode": "0x400F2", + "EventName": "PM_CYC_INST_DISP", + "BriefDescription": "Cycle when instruction(s) dispatched." + }, + { + "EventCode": "0x400F6", + "EventName": "PM_BR_MPRED_CMPL", + "BriefDescription": "A mispredicted branch completed. Includes direction and target." + }, + { + "EventCode": "0x400FA", + "EventName": "PM_RUN_INST_CMPL", + "BriefDescription": "PowerPC instruction completed while the run latch is set." + }, + { + "EventCode": "0x400FC", + "EventName": "PM_ITLB_MISS", + "BriefDescription": "Instruction TLB reload (after a miss), all page sizes. Includes only demand misses." + }, + { + "EventCode": "0x400fe", + "EventName": "PM_LD_NOT_CACHED", + "BriefDescription": "Load data not cached." + }, + { + "EventCode": "0x500fa", + "EventName": "PM_INST_CMPL", + "BriefDescription": "Instructions." + } +] diff --git a/tools/perf/pmu-events/arch/powerpc/mapfile.csv b/tools/perf/pmu-events/arch/powerpc/mapfile.csv index 4d5e9138d4cc..cbd3cb443784 100644 --- a/tools/perf/pmu-events/arch/powerpc/mapfile.csv +++ b/tools/perf/pmu-events/arch/powerpc/mapfile.csv @@ -16,3 +16,4 @@ 0x004e[[:xdigit:]]{4},1,power9,core 0x0080[[:xdigit:]]{4},1,power10,core 0x0082[[:xdigit:]]{4},1,power10,core +0x00ffffff,1,compat,core diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/data-fabric.json b/tools/perf/pmu-events/arch/x86/amdzen5/data-fabric.json new file mode 100644 index 000000000000..fa06569d881d --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/amdzen5/data-fabric.json @@ -0,0 +1,1634 @@ +[ + { + "EventName": "local_or_remote_socket_read_data_beats_dram_0", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 0.", + "EventCode": "0x1f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_1", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 1.", + "EventCode": "0x5f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_2", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 2.", + "EventCode": "0x9f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_3", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 3.", + "EventCode": "0xdf", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_4", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 4.", + "EventCode": "0x11f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_5", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 5.", + "EventCode": "0x15f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_6", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 6.", + "EventCode": "0x19f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_7", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 7.", + "EventCode": "0x1df", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_8", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 8.", + "EventCode": "0x21f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_9", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 9.", + "EventCode": "0x25f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_10", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 10.", + "EventCode": "0x29f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_read_data_beats_dram_11", + "PublicDescription": "Read data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 11.", + "EventCode": "0x2df", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_0", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 0.", + "EventCode": "0x1f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_1", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 1.", + "EventCode": "0x5f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_2", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 2.", + "EventCode": "0x9f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_3", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 3.", + "EventCode": "0xdf", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_4", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 4.", + "EventCode": "0x11f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_5", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 5.", + "EventCode": "0x15f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_6", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 6.", + "EventCode": "0x19f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_7", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 7.", + "EventCode": "0x1df", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_8", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 8.", + "EventCode": "0x21f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_9", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 9.", + "EventCode": "0x25f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_10", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 10.", + "EventCode": "0x29f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_write_data_beats_dram_11", + "PublicDescription": "Write data beats (64 bytes) for transactions between local socket and DRAM Channel 11.", + "EventCode": "0x2df", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_0", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 0.", + "EventCode": "0x1f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_1", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 1.", + "EventCode": "0x5f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_2", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 2.", + "EventCode": "0x9f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_3", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 3.", + "EventCode": "0xdf", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_4", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 4.", + "EventCode": "0x11f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_5", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 5.", + "EventCode": "0x15f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_6", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 6.", + "EventCode": "0x19f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_7", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 7.", + "EventCode": "0x1df", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_8", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 8.", + "EventCode": "0x21f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_9", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 9.", + "EventCode": "0x25f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_10", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 10.", + "EventCode": "0x29f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_write_data_beats_dram_11", + "PublicDescription": "Write data beats (64 bytes) for transactions between remote socket and DRAM Channel 11.", + "EventCode": "0x2df", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_0", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 0.", + "EventCode": "0x1f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_1", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 1.", + "EventCode": "0x5f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_2", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 2.", + "EventCode": "0x9f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_3", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 3.", + "EventCode": "0xdf", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_4", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 4.", + "EventCode": "0x11f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_5", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 5.", + "EventCode": "0x15f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_6", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 6.", + "EventCode": "0x19f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_7", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 7.", + "EventCode": "0x1df", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_8", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 8.", + "EventCode": "0x21f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_9", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 9.", + "EventCode": "0x25f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_10", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 10.", + "EventCode": "0x29f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_write_data_beats_dram_11", + "PublicDescription": "Write data beats (64 bytes) for transactions between local or remote socket and DRAM Channel 11.", + "EventCode": "0x2df", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_0", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 0.", + "EventCode": "0x81f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_1", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 1.", + "EventCode": "0x85f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_2", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 2.", + "EventCode": "0x89f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_3", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 3.", + "EventCode": "0x8df", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_4", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 4.", + "EventCode": "0x91f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_5", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 5.", + "EventCode": "0x95f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_6", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 6.", + "EventCode": "0x99f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_read_data_beats_io_7", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local socket and IO Root Complex 7.", + "EventCode": "0x9df", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_0", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 0.", + "EventCode": "0x81f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_1", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 1.", + "EventCode": "0x85f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_2", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 2.", + "EventCode": "0x89f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_3", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 3.", + "EventCode": "0x8df", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_4", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 4.", + "EventCode": "0x91f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_5", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 5.", + "EventCode": "0x95f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_6", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 6.", + "EventCode": "0x99f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_upstream_write_data_beats_io_7", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local socket and IO Root Complex 7.", + "EventCode": "0x9df", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_0", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 0.", + "EventCode": "0x81f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_1", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 1.", + "EventCode": "0x85f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_2", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 2.", + "EventCode": "0x89f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_3", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 3.", + "EventCode": "0x8df", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_4", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 4.", + "EventCode": "0x91f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_5", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 5.", + "EventCode": "0x95f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_6", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 6.", + "EventCode": "0x99f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_read_data_beats_io_7", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between remote socket and IO Root Complex 7.", + "EventCode": "0x9df", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_0", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 0.", + "EventCode": "0x81f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_1", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 1.", + "EventCode": "0x85f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_2", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 2.", + "EventCode": "0x89f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_3", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 3.", + "EventCode": "0x8df", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_4", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 4.", + "EventCode": "0x91f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_5", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 5.", + "EventCode": "0x95f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_6", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 6.", + "EventCode": "0x99f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_upstream_write_data_beats_io_7", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between remote socket and IO Root Complex 7.", + "EventCode": "0x9df", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_0", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 0.", + "EventCode": "0x81f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_1", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 1.", + "EventCode": "0x85f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_2", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 2.", + "EventCode": "0x89f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_3", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 3.", + "EventCode": "0x8df", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_4", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 4.", + "EventCode": "0x91f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_5", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 5.", + "EventCode": "0x95f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_6", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 6.", + "EventCode": "0x99f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_read_data_beats_io_7", + "PublicDescription": "Upstream DMA read data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 7.", + "EventCode": "0x9df", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_0", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 0.", + "EventCode": "0x81f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_1", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 1.", + "EventCode": "0x85f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_2", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 2.", + "EventCode": "0x89f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_3", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 3.", + "EventCode": "0x8df", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_4", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 4.", + "EventCode": "0x91f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_5", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 5.", + "EventCode": "0x95f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_6", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 6.", + "EventCode": "0x99f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_upstream_write_data_beats_io_7", + "PublicDescription": "Upstream DMA write data beats (64 bytes) for transactions between local or remote socket and IO Root Complex 7.", + "EventCode": "0x9df", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_0", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 0.", + "EventCode": "0x41e", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_1", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 1.", + "EventCode": "0x45e", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_2", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 2.", + "EventCode": "0x49e", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_3", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 3.", + "EventCode": "0x4de", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_4", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 4.", + "EventCode": "0x51e", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_5", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 5.", + "EventCode": "0x55e", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_6", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 6.", + "EventCode": "0x59e", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_7", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 7.", + "EventCode": "0x5de", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_8", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 8.", + "EventCode": "0x41f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_9", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 9.", + "EventCode": "0x45f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_10", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 10.", + "EventCode": "0x49f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_11", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 11.", + "EventCode": "0x4df", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_12", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 12.", + "EventCode": "0x51f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_13", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 13.", + "EventCode": "0x55f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_14", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 14.", + "EventCode": "0x59f", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_cfi_15", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local socket and Core-to-Fabric Interface 15.", + "EventCode": "0x5df", + "UMask": "0x7fe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_0", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 0.", + "EventCode": "0x41e", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_1", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 1.", + "EventCode": "0x45e", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_2", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 2.", + "EventCode": "0x49e", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_3", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 3.", + "EventCode": "0x4de", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_4", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 4.", + "EventCode": "0x51e", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_5", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 5.", + "EventCode": "0x55e", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_6", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 6.", + "EventCode": "0x59e", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_7", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 7.", + "EventCode": "0x5de", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_8", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 8.", + "EventCode": "0x41f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_9", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 9.", + "EventCode": "0x45f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_10", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 10.", + "EventCode": "0x49f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_11", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 11.", + "EventCode": "0x4df", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_12", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 12.", + "EventCode": "0x51f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_13", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 13.", + "EventCode": "0x55f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_14", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 14.", + "EventCode": "0x59f", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_cfi_15", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and Core-to-Fabric Interface 15.", + "EventCode": "0x5df", + "UMask": "0x7ff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_0", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 0.", + "EventCode": "0x41e", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_1", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 1.", + "EventCode": "0x45e", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_2", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 2.", + "EventCode": "0x49e", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_3", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 3.", + "EventCode": "0x4de", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_4", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 4.", + "EventCode": "0x51e", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_5", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 5.", + "EventCode": "0x55e", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_6", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 6.", + "EventCode": "0x59e", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_7", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 7.", + "EventCode": "0x5de", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_8", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 8.", + "EventCode": "0x41f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_9", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 9.", + "EventCode": "0x45f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_10", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 10.", + "EventCode": "0x49f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_11", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 11.", + "EventCode": "0x4df", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_12", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 12.", + "EventCode": "0x51f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_13", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 13.", + "EventCode": "0x55f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_14", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 14.", + "EventCode": "0x59f", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_inbound_data_beats_cfi_15", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between remote socket and Core-to-Fabric Interface 15.", + "EventCode": "0x5df", + "UMask": "0xbfe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_0", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 0.", + "EventCode": "0x41e", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_1", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 1.", + "EventCode": "0x45e", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_2", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 2.", + "EventCode": "0x49e", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_3", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 3.", + "EventCode": "0x4de", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_4", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 4.", + "EventCode": "0x51e", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_5", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 5.", + "EventCode": "0x55e", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_6", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 6.", + "EventCode": "0x59e", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_7", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 7.", + "EventCode": "0x5de", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_8", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 8.", + "EventCode": "0x41f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_9", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 9.", + "EventCode": "0x45f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_10", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 10.", + "EventCode": "0x49f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_11", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 11.", + "EventCode": "0x4df", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_12", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 12.", + "EventCode": "0x51f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_13", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 13.", + "EventCode": "0x55f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_14", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 14.", + "EventCode": "0x59f", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "remote_socket_outbound_data_beats_cfi_15", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between remote socket and Core-to-Fabric Interface 15.", + "EventCode": "0x5df", + "UMask": "0xbff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_0", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 0.", + "EventCode": "0x41e", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_1", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 1.", + "EventCode": "0x45e", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_2", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 2.", + "EventCode": "0x49e", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_3", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 3.", + "EventCode": "0x4de", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_4", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 4.", + "EventCode": "0x51e", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_5", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 5.", + "EventCode": "0x55e", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_6", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 6.", + "EventCode": "0x59e", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_7", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 7.", + "EventCode": "0x5de", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_8", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 8.", + "EventCode": "0x41f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_9", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 9.", + "EventCode": "0x45f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_10", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 10.", + "EventCode": "0x49f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_11", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 11.", + "EventCode": "0x4df", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_12", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 12.", + "EventCode": "0x51f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_13", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 13.", + "EventCode": "0x55f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_14", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 14.", + "EventCode": "0x59f", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_inbound_data_beats_cfi_15", + "PublicDescription": "Inbound data beats (32 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 15.", + "EventCode": "0x5df", + "UMask": "0xffe", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_0", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 0.", + "EventCode": "0x41e", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_1", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 1.", + "EventCode": "0x45e", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_2", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 2.", + "EventCode": "0x49e", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_3", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 3.", + "EventCode": "0x4de", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_4", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 4.", + "EventCode": "0x51e", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_5", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 5.", + "EventCode": "0x55e", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_6", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 6.", + "EventCode": "0x59e", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_7", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 7.", + "EventCode": "0x5de", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_8", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 8.", + "EventCode": "0x41f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_9", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 9.", + "EventCode": "0x45f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_10", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 10.", + "EventCode": "0x49f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_11", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 11.", + "EventCode": "0x4df", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_12", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 12.", + "EventCode": "0x51f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_13", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 13.", + "EventCode": "0x55f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_14", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 14.", + "EventCode": "0x59f", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_or_remote_socket_outbound_data_beats_cfi_15", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local or remote socket and Core-to-Fabric Interface 15.", + "EventCode": "0x5df", + "UMask": "0xfff", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_link_0", + "PublicDescription": "Inbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 0.", + "EventCode": "0xd5f", + "UMask": "0xf3f", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_link_1", + "PublicDescription": "Inbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 1.", + "EventCode": "0xd9f", + "UMask": "0xf3f", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_link_2", + "PublicDescription": "Inbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 2.", + "EventCode": "0xddf", + "UMask": "0xf3f", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_link_3", + "PublicDescription": "Inbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 3.", + "EventCode": "0xe1f", + "UMask": "0xf3f", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_link_4", + "PublicDescription": "Inbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 4.", + "EventCode": "0xe5f", + "UMask": "0xf3f", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_inbound_data_beats_link_5", + "PublicDescription": "Inbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 5.", + "EventCode": "0xe9f", + "UMask": "0xf3f", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_link_0", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 0.", + "EventCode": "0xd5f", + "UMask": "0xf3e", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_link_1", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 1.", + "EventCode": "0xd9f", + "UMask": "0xf3e", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_link_2", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 2.", + "EventCode": "0xddf", + "UMask": "0xf3e", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_link_3", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 3.", + "EventCode": "0xe1f", + "UMask": "0xf3e", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_link_4", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 4.", + "EventCode": "0xe5f", + "UMask": "0xf3e", + "PerPkg": "1", + "Unit": "DFPMC" + }, + { + "EventName": "local_socket_outbound_data_beats_link_5", + "PublicDescription": "Outbound data beats (64 bytes) for transactions between local socket and remote socket over Cross-socket Link 5.", + "EventCode": "0xe9f", + "UMask": "0xf3e", + "PerPkg": "1", + "Unit": "DFPMC" + } +] diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index af2fdf1f55d6..ff6627a77805 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -97,6 +97,12 @@ "UMask": "0x02" }, { + "EventName": "ls_dmnd_fills_from_sys.local_all", + "EventCode": "0x43", + "BriefDescription": "Demand data cache fills from local L2 cache, L3 cache or different L2 cache in the same CCX.", + "UMask": "0x03" + }, + { "EventName": "ls_dmnd_fills_from_sys.near_cache", "EventCode": "0x43", "BriefDescription": "Demand data cache fills from cache of another CCX when the address was in the same NUMA node.", @@ -115,12 +121,30 @@ "UMask": "0x10" }, { + "EventName": "ls_dmnd_fills_from_sys.remote_cache", + "EventCode": "0x43", + "BriefDescription": "Demand data cache fills from cache of another CCX when the address was in the same or a different NUMA node.", + "UMask": "0x14" + }, + { "EventName": "ls_dmnd_fills_from_sys.dram_io_far", "EventCode": "0x43", "BriefDescription": "Demand data cache fills from either DRAM or MMIO in a different NUMA node (same or different socket).", "UMask": "0x40" }, { + "EventName": "ls_dmnd_fills_from_sys.dram_io_all", + "EventCode": "0x43", + "BriefDescription": "Demand data cache fills from either DRAM or MMIO in the same or a different NUMA node (same or different socket).", + "UMask": "0x48" + }, + { + "EventName": "ls_dmnd_fills_from_sys.far_all", + "EventCode": "0x43", + "BriefDescription": "Demand data cache fills from either cache of another CCX, DRAM or MMIO when the address was in a different NUMA node (same or different socket).", + "UMask": "0x50" + }, + { "EventName": "ls_dmnd_fills_from_sys.alternate_memories", "EventCode": "0x43", "BriefDescription": "Demand data cache fills from extension memory.", @@ -193,12 +217,6 @@ "UMask": "0x50" }, { - "EventName": "ls_any_fills_from_sys.all_dram_io", - "EventCode": "0x44", - "BriefDescription": "Any data cache fills from either DRAM or MMIO in any NUMA node (same or different socket).", - "UMask": "0x48" - }, - { "EventName": "ls_any_fills_from_sys.alternate_memories", "EventCode": "0x44", "BriefDescription": "Any data cache fills from extension memory.", @@ -343,6 +361,12 @@ "UMask": "0x02" }, { + "EventName": "ls_sw_pf_dc_fills.local_all", + "EventCode": "0x59", + "BriefDescription": "Software prefetch data cache fills from local L2 cache, L3 cache or different L2 cache in the same CCX.", + "UMask": "0x03" + }, + { "EventName": "ls_sw_pf_dc_fills.near_cache", "EventCode": "0x59", "BriefDescription": "Software prefetch data cache fills from cache of another CCX in the same NUMA node.", @@ -361,12 +385,30 @@ "UMask": "0x10" }, { + "EventName": "ls_sw_pf_dc_fills.remote_cache", + "EventCode": "0x59", + "BriefDescription": "Software prefetch data cache fills from cache of another CCX when the address was in the same or a different NUMA node.", + "UMask": "0x14" + }, + { "EventName": "ls_sw_pf_dc_fills.dram_io_far", "EventCode": "0x59", "BriefDescription": "Software prefetch data cache fills from either DRAM or MMIO in a different NUMA node (same or different socket).", "UMask": "0x40" }, { + "EventName": "ls_sw_pf_dc_fills.dram_io_all", + "EventCode": "0x59", + "BriefDescription": "Software prefetch data cache fills from either DRAM or MMIO in the same or a different NUMA node (same or different socket).", + "UMask": "0x48" + }, + { + "EventName": "ls_sw_pf_dc_fills.far_all", + "EventCode": "0x59", + "BriefDescription": "Software prefetch data cache fills from either cache of another CCX, DRAM or MMIO when the address was in a different NUMA node (same or different socket).", + "UMask": "0x50" + }, + { "EventName": "ls_sw_pf_dc_fills.alternate_memories", "EventCode": "0x59", "BriefDescription": "Software prefetch data cache fills from extension memory.", @@ -391,6 +433,12 @@ "UMask": "0x02" }, { + "EventName": "ls_hw_pf_dc_fills.local_all", + "EventCode": "0x5a", + "BriefDescription": "Hardware prefetch data cache fills from local L2 cache, L3 cache or different L2 cache in the same CCX.", + "UMask": "0x03" + }, + { "EventName": "ls_hw_pf_dc_fills.near_cache", "EventCode": "0x5a", "BriefDescription": "Hardware prefetch data cache fills from cache of another CCX when the address was in the same NUMA node.", @@ -409,12 +457,30 @@ "UMask": "0x10" }, { + "EventName": "ls_hw_pf_dc_fills.remote_cache", + "EventCode": "0x5a", + "BriefDescription": "Hardware prefetch data cache fills from cache of another CCX when the address was in the same or a different NUMA node.", + "UMask": "0x14" + }, + { "EventName": "ls_hw_pf_dc_fills.dram_io_far", "EventCode": "0x5a", "BriefDescription": "Hardware prefetch data cache fills from either DRAM or MMIO in a different NUMA node (same or different socket).", "UMask": "0x40" }, { + "EventName": "ls_hw_pf_dc_fills.dram_io_all", + "EventCode": "0x5a", + "BriefDescription": "Hardware prefetch data cache fills from either DRAM or MMIO in the same or a different NUMA node (same or different socket).", + "UMask": "0x48" + }, + { + "EventName": "ls_hw_pf_dc_fills.far_all", + "EventCode": "0x5a", + "BriefDescription": "Hardware prefetch data cache fills from either cache of another CCX, DRAM or MMIO when the address was in a different NUMA node (same or different socket).", + "UMask": "0x50" + }, + { "EventName": "ls_hw_pf_dc_fills.alternate_memories", "EventCode": "0x5a", "BriefDescription": "Hardware prefetch data cache fills from extension memory.", diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/recommended.json b/tools/perf/pmu-events/arch/x86/amdzen5/recommended.json index c97874039c1e..635d57e3bc15 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/recommended.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/recommended.json @@ -341,5 +341,117 @@ "MetricGroup": "memory_controller", "PerPkg": "1", "ScaleUnit": "1per_memclk" + }, + { + "MetricName": "dram_read_bandwidth_for_local_or_remote_socket", + "BriefDescription": "DRAM read data bandwidth for accesses in local or remote socket.", + "MetricExpr": "(local_or_remote_socket_read_data_beats_dram_0 + local_or_remote_socket_read_data_beats_dram_1 + local_or_remote_socket_read_data_beats_dram_2 + local_or_remote_socket_read_data_beats_dram_3 + local_or_remote_socket_read_data_beats_dram_4 + local_or_remote_socket_read_data_beats_dram_5 + local_or_remote_socket_read_data_beats_dram_6 + local_or_remote_socket_read_data_beats_dram_7 + local_or_remote_socket_read_data_beats_dram_8 + local_or_remote_socket_read_data_beats_dram_9 + local_or_remote_socket_read_data_beats_dram_10 + local_or_remote_socket_read_data_beats_dram_11) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "dram_write_bandwidth_for_local_socket", + "BriefDescription": "DRAM write data bandwidth for accesses in local socket.", + "MetricExpr": "(local_socket_write_data_beats_dram_0 + local_socket_write_data_beats_dram_1 + local_socket_write_data_beats_dram_2 + local_socket_write_data_beats_dram_3 + local_socket_write_data_beats_dram_4 + local_socket_write_data_beats_dram_5 + local_socket_write_data_beats_dram_6 + local_socket_write_data_beats_dram_7 + local_socket_write_data_beats_dram_8 + local_socket_write_data_beats_dram_9 + local_socket_write_data_beats_dram_10 + local_socket_write_data_beats_dram_11) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "dram_write_bandwidth_for_remote_socket", + "BriefDescription": "DRAM write data bandwidth for accesses in remote socket.", + "MetricExpr": "(remote_socket_write_data_beats_dram_0 + remote_socket_write_data_beats_dram_1 + remote_socket_write_data_beats_dram_2 + remote_socket_write_data_beats_dram_3 + remote_socket_write_data_beats_dram_4 + remote_socket_write_data_beats_dram_5 + remote_socket_write_data_beats_dram_6 + remote_socket_write_data_beats_dram_7 + remote_socket_write_data_beats_dram_8 + remote_socket_write_data_beats_dram_9 + remote_socket_write_data_beats_dram_10 + remote_socket_write_data_beats_dram_11) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "dram_write_bandwidth_for_local_or_remote_socket", + "BriefDescription": "DRAM write data bandwidth for accesses in local or remote socket.", + "MetricExpr": "(local_or_remote_socket_write_data_beats_dram_0 + local_or_remote_socket_write_data_beats_dram_1 + local_or_remote_socket_write_data_beats_dram_2 + local_or_remote_socket_write_data_beats_dram_3 + local_or_remote_socket_write_data_beats_dram_4 + local_or_remote_socket_write_data_beats_dram_5 + local_or_remote_socket_write_data_beats_dram_6 + local_or_remote_socket_write_data_beats_dram_7 + local_or_remote_socket_write_data_beats_dram_8 + local_or_remote_socket_write_data_beats_dram_9 + local_or_remote_socket_write_data_beats_dram_10 + local_or_remote_socket_write_data_beats_dram_11) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "upstream_dma_read_bandwidth_for_local_socket", + "BriefDescription": "Upstream DMA read data bandwidth for accesses in local socket.", + "MetricExpr": "(local_socket_upstream_read_data_beats_io_0 + local_socket_upstream_read_data_beats_io_1 + local_socket_upstream_read_data_beats_io_2 + local_socket_upstream_read_data_beats_io_3 + local_socket_upstream_read_data_beats_io_4 + local_socket_upstream_read_data_beats_io_5 + local_socket_upstream_read_data_beats_io_6 + local_socket_upstream_read_data_beats_io_7) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "upstream_dma_write_bandwidth_for_local_socket", + "BriefDescription": "Upstream DMA write data bandwidth for accesses in local socket.", + "MetricExpr": "(local_socket_upstream_write_data_beats_io_0 + local_socket_upstream_write_data_beats_io_1 + local_socket_upstream_write_data_beats_io_2 + local_socket_upstream_write_data_beats_io_3 + local_socket_upstream_write_data_beats_io_4 + local_socket_upstream_write_data_beats_io_5 + local_socket_upstream_write_data_beats_io_6 + local_socket_upstream_write_data_beats_io_7) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "upstream_dma_read_bandwidth_for_remote_socket", + "BriefDescription": "Upstream DMA read data bandwidth for accesses in remote socket.", + "MetricExpr": "(remote_socket_upstream_read_data_beats_io_0 + remote_socket_upstream_read_data_beats_io_1 + remote_socket_upstream_read_data_beats_io_2 + remote_socket_upstream_read_data_beats_io_3 + remote_socket_upstream_read_data_beats_io_4 + remote_socket_upstream_read_data_beats_io_5 + remote_socket_upstream_read_data_beats_io_6 + remote_socket_upstream_read_data_beats_io_7) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "upstream_dma_write_bandwidth_for_remote_socket", + "BriefDescription": "Upstream DMA write data bandwidth for accesses in remote socket.", + "MetricExpr": "(remote_socket_upstream_write_data_beats_io_0 + remote_socket_upstream_write_data_beats_io_1 + remote_socket_upstream_write_data_beats_io_2 + remote_socket_upstream_write_data_beats_io_3 + remote_socket_upstream_write_data_beats_io_4 + remote_socket_upstream_write_data_beats_io_5 + remote_socket_upstream_write_data_beats_io_6 + remote_socket_upstream_write_data_beats_io_7) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "core_inbound_data_bandwidth_for_local_socket", + "BriefDescription": "Core inbound data bandwidth for accesses in local socket.", + "MetricExpr": "(local_socket_inbound_data_beats_cfi_0 + local_socket_inbound_data_beats_cfi_1 + local_socket_inbound_data_beats_cfi_2 + local_socket_inbound_data_beats_cfi_3 + local_socket_inbound_data_beats_cfi_4 + local_socket_inbound_data_beats_cfi_5 + local_socket_inbound_data_beats_cfi_6 + local_socket_inbound_data_beats_cfi_7 + local_socket_inbound_data_beats_cfi_8 + local_socket_inbound_data_beats_cfi_9 + local_socket_inbound_data_beats_cfi_10 + local_socket_inbound_data_beats_cfi_11 + local_socket_inbound_data_beats_cfi_12 + local_socket_inbound_data_beats_cfi_13 + local_socket_inbound_data_beats_cfi_14 + local_socket_inbound_data_beats_cfi_15) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "3.2e-5MB/s" + }, + { + "MetricName": "core_outbound_data_bandwidth_for_local_socket", + "BriefDescription": "Core outbound data bandwidth for accesses in local socket.", + "MetricExpr": "(local_socket_outbound_data_beats_cfi_0 + local_socket_outbound_data_beats_cfi_1 + local_socket_outbound_data_beats_cfi_2 + local_socket_outbound_data_beats_cfi_3 + local_socket_outbound_data_beats_cfi_4 + local_socket_outbound_data_beats_cfi_5 + local_socket_outbound_data_beats_cfi_6 + local_socket_outbound_data_beats_cfi_7 + local_socket_outbound_data_beats_cfi_8 + local_socket_outbound_data_beats_cfi_9 + local_socket_outbound_data_beats_cfi_10 + local_socket_outbound_data_beats_cfi_11 + local_socket_outbound_data_beats_cfi_12 + local_socket_outbound_data_beats_cfi_13 + local_socket_outbound_data_beats_cfi_14 + local_socket_outbound_data_beats_cfi_15) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "core_inbound_data_bandwidth_for_remote_socket", + "BriefDescription": "Core inbound data bandwidth for accesses in remote socket.", + "MetricExpr": "(remote_socket_inbound_data_beats_cfi_0 + remote_socket_inbound_data_beats_cfi_1 + remote_socket_inbound_data_beats_cfi_2 + remote_socket_inbound_data_beats_cfi_3 + remote_socket_inbound_data_beats_cfi_4 + remote_socket_inbound_data_beats_cfi_5 + remote_socket_inbound_data_beats_cfi_6 + remote_socket_inbound_data_beats_cfi_7 + remote_socket_inbound_data_beats_cfi_8 + remote_socket_inbound_data_beats_cfi_9 + remote_socket_inbound_data_beats_cfi_10 + remote_socket_inbound_data_beats_cfi_11 + remote_socket_inbound_data_beats_cfi_12 + remote_socket_inbound_data_beats_cfi_13 + remote_socket_inbound_data_beats_cfi_14 + remote_socket_inbound_data_beats_cfi_15) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "3.2e-5MB/s" + }, + { + "MetricName": "core_outbound_data_bandwidth_for_remote_socket", + "BriefDescription": "Core outbound data bandwidth for accesses in remote socket.", + "MetricExpr": "(remote_socket_outbound_data_beats_cfi_0 + remote_socket_outbound_data_beats_cfi_1 + remote_socket_outbound_data_beats_cfi_2 + remote_socket_outbound_data_beats_cfi_3 + remote_socket_outbound_data_beats_cfi_4 + remote_socket_outbound_data_beats_cfi_5 + remote_socket_outbound_data_beats_cfi_6 + remote_socket_outbound_data_beats_cfi_7 + remote_socket_outbound_data_beats_cfi_8 + remote_socket_outbound_data_beats_cfi_9 + remote_socket_outbound_data_beats_cfi_10 + remote_socket_outbound_data_beats_cfi_11 + remote_socket_outbound_data_beats_cfi_12 + remote_socket_outbound_data_beats_cfi_13 + remote_socket_outbound_data_beats_cfi_14 + remote_socket_outbound_data_beats_cfi_15) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "cross_socket_inbound_data_bandwidth_for_local_socket", + "BriefDescription": "Inbound data bandwidth for accesses between local socket and remote socket.", + "MetricExpr": "(local_socket_inbound_data_beats_link_0 + local_socket_inbound_data_beats_link_1 + local_socket_inbound_data_beats_link_2 + local_socket_inbound_data_beats_link_3 + local_socket_inbound_data_beats_link_4 + local_socket_inbound_data_beats_link_5) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" + }, + { + "MetricName": "cross_socket_outbound_data_bandwidth_for_local_socket", + "BriefDescription": "Outbound data bandwidth for accesses between local socket and remote socket.", + "MetricExpr": "(local_socket_outbound_data_beats_link_0 + local_socket_outbound_data_beats_link_1 + local_socket_outbound_data_beats_link_2 + local_socket_outbound_data_beats_link_3 + local_socket_outbound_data_beats_link_4 + local_socket_outbound_data_beats_link_5) / duration_time", + "MetricGroup": "data_fabric", + "PerPkg": "1", + "ScaleUnit": "6.4e-5MB/s" } ] diff --git a/tools/perf/pmu-events/empty-pmu-events.c b/tools/perf/pmu-events/empty-pmu-events.c index c592079982fb..1c7a2cfa321f 100644 --- a/tools/perf/pmu-events/empty-pmu-events.c +++ b/tools/perf/pmu-events/empty-pmu-events.c @@ -19,72 +19,109 @@ struct pmu_table_entry { }; static const char *const big_c_string = -/* offset=0 */ "default_core\000" -/* offset=13 */ "bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000" -/* offset=72 */ "bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000" -/* offset=131 */ "l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000Attributable Level 3 cache access, read\000" -/* offset=226 */ "segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000" -/* offset=325 */ "dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000" -/* offset=455 */ "eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000" -/* offset=570 */ "hisi_sccl,ddrc\000" -/* offset=585 */ "uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000DDRC write commands\000" -/* offset=671 */ "uncore_cbox\000" -/* offset=683 */ "unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000" -/* offset=914 */ "event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000UNC_CBO_HYPHEN\000" -/* offset=979 */ "event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000UNC_CBO_TWO_HYPH\000" -/* offset=1050 */ "hisi_sccl,l3c\000" -/* offset=1064 */ "uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000Total read hits\000" -/* offset=1144 */ "uncore_imc_free_running\000" -/* offset=1168 */ "uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000Total cache misses\000" -/* offset=1263 */ "uncore_imc\000" -/* offset=1274 */ "uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000Total cache hits\000" -/* offset=1352 */ "uncore_sys_ddr_pmu\000" -/* offset=1371 */ "sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000" -/* offset=1444 */ "uncore_sys_ccn_pmu\000" -/* offset=1463 */ "sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000" -/* offset=1537 */ "uncore_sys_cmn_pmu\000" -/* offset=1556 */ "sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000" -/* offset=1696 */ "CPI\000\0001 / IPC\000\000\000\000\000\000\000\00000" -/* offset=1718 */ "IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\00000" -/* offset=1781 */ "Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\00000" -/* offset=1947 */ "dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000" -/* offset=2011 */ "icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000" -/* offset=2078 */ "cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\00000" -/* offset=2149 */ "DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\00000" -/* offset=2243 */ "DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\00000" -/* offset=2377 */ "DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\00000" -/* offset=2441 */ "DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\00000" -/* offset=2509 */ "DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\00000" -/* offset=2579 */ "M1\000\000ipc + M2\000\000\000\000\000\000\000\00000" -/* offset=2601 */ "M2\000\000ipc + M1\000\000\000\000\000\000\000\00000" -/* offset=2623 */ "M3\000\0001 / M3\000\000\000\000\000\000\000\00000" -/* offset=2643 */ "L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\00000" +/* offset=0 */ "tool\000" +/* offset=5 */ "duration_time\000tool\000Wall clock interval time in nanoseconds\000config=1\000\00000\000\000" +/* offset=78 */ "user_time\000tool\000User (non-kernel) time in nanoseconds\000config=2\000\00000\000\000" +/* offset=145 */ "system_time\000tool\000System/kernel time in nanoseconds\000config=3\000\00000\000\000" +/* offset=210 */ "has_pmem\000tool\0001 if persistent memory installed otherwise 0\000config=4\000\00000\000\000" +/* offset=283 */ "num_cores\000tool\000Number of cores. A core consists of 1 or more thread, with each thread being associated with a logical Linux CPU\000config=5\000\00000\000\000" +/* offset=425 */ "num_cpus\000tool\000Number of logical Linux CPUs. There may be multiple such CPUs on a core\000config=6\000\00000\000\000" +/* offset=525 */ "num_cpus_online\000tool\000Number of online logical Linux CPUs. There may be multiple such CPUs on a core\000config=7\000\00000\000\000" +/* offset=639 */ "num_dies\000tool\000Number of dies. Each die has 1 or more cores\000config=8\000\00000\000\000" +/* offset=712 */ "num_packages\000tool\000Number of packages. Each package has 1 or more die\000config=9\000\00000\000\000" +/* offset=795 */ "slots\000tool\000Number of functional units that in parallel can execute parts of an instruction\000config=0xa\000\00000\000\000" +/* offset=902 */ "smt_on\000tool\0001 if simultaneous multithreading (aka hyperthreading) is enable otherwise 0\000config=0xb\000\00000\000\000" +/* offset=1006 */ "system_tsc_freq\000tool\000The amount a Time Stamp Counter (TSC) increases per second\000config=0xc\000\00000\000\000" +/* offset=1102 */ "default_core\000" +/* offset=1115 */ "bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000" +/* offset=1174 */ "bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000" +/* offset=1233 */ "l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000Attributable Level 3 cache access, read\000" +/* offset=1328 */ "segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000" +/* offset=1427 */ "dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000" +/* offset=1557 */ "eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000" +/* offset=1672 */ "hisi_sccl,ddrc\000" +/* offset=1687 */ "uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000DDRC write commands\000" +/* offset=1773 */ "uncore_cbox\000" +/* offset=1785 */ "unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000" +/* offset=2016 */ "event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000UNC_CBO_HYPHEN\000" +/* offset=2081 */ "event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000UNC_CBO_TWO_HYPH\000" +/* offset=2152 */ "hisi_sccl,l3c\000" +/* offset=2166 */ "uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000Total read hits\000" +/* offset=2246 */ "uncore_imc_free_running\000" +/* offset=2270 */ "uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000Total cache misses\000" +/* offset=2365 */ "uncore_imc\000" +/* offset=2376 */ "uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000Total cache hits\000" +/* offset=2454 */ "uncore_sys_ddr_pmu\000" +/* offset=2473 */ "sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000" +/* offset=2546 */ "uncore_sys_ccn_pmu\000" +/* offset=2565 */ "sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000" +/* offset=2639 */ "uncore_sys_cmn_pmu\000" +/* offset=2658 */ "sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000" +/* offset=2798 */ "CPI\000\0001 / IPC\000\000\000\000\000\000\000\00000" +/* offset=2820 */ "IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\00000" +/* offset=2883 */ "Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\00000" +/* offset=3049 */ "dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000" +/* offset=3113 */ "icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000" +/* offset=3180 */ "cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\00000" +/* offset=3251 */ "DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\00000" +/* offset=3345 */ "DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\00000" +/* offset=3479 */ "DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\00000" +/* offset=3543 */ "DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\00000" +/* offset=3611 */ "DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\00000" +/* offset=3681 */ "M1\000\000ipc + M2\000\000\000\000\000\000\000\00000" +/* offset=3703 */ "M2\000\000ipc + M1\000\000\000\000\000\000\000\00000" +/* offset=3725 */ "M3\000\0001 / M3\000\000\000\000\000\000\000\00000" +/* offset=3745 */ "L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\00000" ; +static const struct compact_pmu_event pmu_events__common_tool[] = { +{ 5 }, /* duration_time\000tool\000Wall clock interval time in nanoseconds\000config=1\000\00000\000\000 */ +{ 210 }, /* has_pmem\000tool\0001 if persistent memory installed otherwise 0\000config=4\000\00000\000\000 */ +{ 283 }, /* num_cores\000tool\000Number of cores. A core consists of 1 or more thread, with each thread being associated with a logical Linux CPU\000config=5\000\00000\000\000 */ +{ 425 }, /* num_cpus\000tool\000Number of logical Linux CPUs. There may be multiple such CPUs on a core\000config=6\000\00000\000\000 */ +{ 525 }, /* num_cpus_online\000tool\000Number of online logical Linux CPUs. There may be multiple such CPUs on a core\000config=7\000\00000\000\000 */ +{ 639 }, /* num_dies\000tool\000Number of dies. Each die has 1 or more cores\000config=8\000\00000\000\000 */ +{ 712 }, /* num_packages\000tool\000Number of packages. Each package has 1 or more die\000config=9\000\00000\000\000 */ +{ 795 }, /* slots\000tool\000Number of functional units that in parallel can execute parts of an instruction\000config=0xa\000\00000\000\000 */ +{ 902 }, /* smt_on\000tool\0001 if simultaneous multithreading (aka hyperthreading) is enable otherwise 0\000config=0xb\000\00000\000\000 */ +{ 145 }, /* system_time\000tool\000System/kernel time in nanoseconds\000config=3\000\00000\000\000 */ +{ 1006 }, /* system_tsc_freq\000tool\000The amount a Time Stamp Counter (TSC) increases per second\000config=0xc\000\00000\000\000 */ +{ 78 }, /* user_time\000tool\000User (non-kernel) time in nanoseconds\000config=2\000\00000\000\000 */ + +}; + +const struct pmu_table_entry pmu_events__common[] = { +{ + .entries = pmu_events__common_tool, + .num_entries = ARRAY_SIZE(pmu_events__common_tool), + .pmu_name = { 0 /* tool\000 */ }, +}, +}; + static const struct compact_pmu_event pmu_events__test_soc_cpu_default_core[] = { -{ 13 }, /* bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000 */ -{ 72 }, /* bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000 */ -{ 325 }, /* dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000 */ -{ 455 }, /* eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000 */ -{ 131 }, /* l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000Attributable Level 3 cache access, read\000 */ -{ 226 }, /* segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000 */ +{ 1115 }, /* bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000 */ +{ 1174 }, /* bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000 */ +{ 1427 }, /* dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000 */ +{ 1557 }, /* eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000 */ +{ 1233 }, /* l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000Attributable Level 3 cache access, read\000 */ +{ 1328 }, /* segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_cpu_hisi_sccl_ddrc[] = { -{ 585 }, /* uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000DDRC write commands\000 */ +{ 1687 }, /* uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000DDRC write commands\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_cpu_hisi_sccl_l3c[] = { -{ 1064 }, /* uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000Total read hits\000 */ +{ 2166 }, /* uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000Total read hits\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_cpu_uncore_cbox[] = { -{ 914 }, /* event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000UNC_CBO_HYPHEN\000 */ -{ 979 }, /* event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000UNC_CBO_TWO_HYPH\000 */ -{ 683 }, /* unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000 */ +{ 2016 }, /* event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000UNC_CBO_HYPHEN\000 */ +{ 2081 }, /* event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000UNC_CBO_TWO_HYPH\000 */ +{ 1785 }, /* unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_cpu_uncore_imc[] = { -{ 1274 }, /* uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000Total cache hits\000 */ +{ 2376 }, /* uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000Total cache hits\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_cpu_uncore_imc_free_running[] = { -{ 1168 }, /* uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000Total cache misses\000 */ +{ 2270 }, /* uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000Total cache misses\000 */ }; @@ -92,51 +129,51 @@ const struct pmu_table_entry pmu_events__test_soc_cpu[] = { { .entries = pmu_events__test_soc_cpu_default_core, .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_default_core), - .pmu_name = { 0 /* default_core\000 */ }, + .pmu_name = { 1102 /* default_core\000 */ }, }, { .entries = pmu_events__test_soc_cpu_hisi_sccl_ddrc, .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_hisi_sccl_ddrc), - .pmu_name = { 570 /* hisi_sccl,ddrc\000 */ }, + .pmu_name = { 1672 /* hisi_sccl,ddrc\000 */ }, }, { .entries = pmu_events__test_soc_cpu_hisi_sccl_l3c, .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_hisi_sccl_l3c), - .pmu_name = { 1050 /* hisi_sccl,l3c\000 */ }, + .pmu_name = { 2152 /* hisi_sccl,l3c\000 */ }, }, { .entries = pmu_events__test_soc_cpu_uncore_cbox, .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_cbox), - .pmu_name = { 671 /* uncore_cbox\000 */ }, + .pmu_name = { 1773 /* uncore_cbox\000 */ }, }, { .entries = pmu_events__test_soc_cpu_uncore_imc, .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_imc), - .pmu_name = { 1263 /* uncore_imc\000 */ }, + .pmu_name = { 2365 /* uncore_imc\000 */ }, }, { .entries = pmu_events__test_soc_cpu_uncore_imc_free_running, .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_imc_free_running), - .pmu_name = { 1144 /* uncore_imc_free_running\000 */ }, + .pmu_name = { 2246 /* uncore_imc_free_running\000 */ }, }, }; static const struct compact_pmu_event pmu_metrics__test_soc_cpu_default_core[] = { -{ 1696 }, /* CPI\000\0001 / IPC\000\000\000\000\000\000\000\00000 */ -{ 2377 }, /* DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\00000 */ -{ 2149 }, /* DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\00000 */ -{ 2243 }, /* DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\00000 */ -{ 2441 }, /* DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\00000 */ -{ 2509 }, /* DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\00000 */ -{ 1781 }, /* Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\00000 */ -{ 1718 }, /* IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\00000 */ -{ 2643 }, /* L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\00000 */ -{ 2579 }, /* M1\000\000ipc + M2\000\000\000\000\000\000\000\00000 */ -{ 2601 }, /* M2\000\000ipc + M1\000\000\000\000\000\000\000\00000 */ -{ 2623 }, /* M3\000\0001 / M3\000\000\000\000\000\000\000\00000 */ -{ 2078 }, /* cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\00000 */ -{ 1947 }, /* dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000 */ -{ 2011 }, /* icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000 */ +{ 2798 }, /* CPI\000\0001 / IPC\000\000\000\000\000\000\000\00000 */ +{ 3479 }, /* DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\00000 */ +{ 3251 }, /* DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\00000 */ +{ 3345 }, /* DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\00000 */ +{ 3543 }, /* DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\00000 */ +{ 3611 }, /* DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\00000 */ +{ 2883 }, /* Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\00000 */ +{ 2820 }, /* IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\00000 */ +{ 3745 }, /* L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\00000 */ +{ 3681 }, /* M1\000\000ipc + M2\000\000\000\000\000\000\000\00000 */ +{ 3703 }, /* M2\000\000ipc + M1\000\000\000\000\000\000\000\00000 */ +{ 3725 }, /* M3\000\0001 / M3\000\000\000\000\000\000\000\00000 */ +{ 3180 }, /* cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\00000 */ +{ 3049 }, /* dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000 */ +{ 3113 }, /* icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\00000 */ }; @@ -144,18 +181,18 @@ const struct pmu_table_entry pmu_metrics__test_soc_cpu[] = { { .entries = pmu_metrics__test_soc_cpu_default_core, .num_entries = ARRAY_SIZE(pmu_metrics__test_soc_cpu_default_core), - .pmu_name = { 0 /* default_core\000 */ }, + .pmu_name = { 1102 /* default_core\000 */ }, }, }; static const struct compact_pmu_event pmu_events__test_soc_sys_uncore_sys_ccn_pmu[] = { -{ 1463 }, /* sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000 */ +{ 2565 }, /* sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_sys_uncore_sys_cmn_pmu[] = { -{ 1556 }, /* sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000 */ +{ 2658 }, /* sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000 */ }; static const struct compact_pmu_event pmu_events__test_soc_sys_uncore_sys_ddr_pmu[] = { -{ 1371 }, /* sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000 */ +{ 2473 }, /* sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000 */ }; @@ -163,17 +200,17 @@ const struct pmu_table_entry pmu_events__test_soc_sys[] = { { .entries = pmu_events__test_soc_sys_uncore_sys_ccn_pmu, .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_ccn_pmu), - .pmu_name = { 1444 /* uncore_sys_ccn_pmu\000 */ }, + .pmu_name = { 2546 /* uncore_sys_ccn_pmu\000 */ }, }, { .entries = pmu_events__test_soc_sys_uncore_sys_cmn_pmu, .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_cmn_pmu), - .pmu_name = { 1537 /* uncore_sys_cmn_pmu\000 */ }, + .pmu_name = { 2639 /* uncore_sys_cmn_pmu\000 */ }, }, { .entries = pmu_events__test_soc_sys_uncore_sys_ddr_pmu, .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_ddr_pmu), - .pmu_name = { 1352 /* uncore_sys_ddr_pmu\000 */ }, + .pmu_name = { 2454 /* uncore_sys_ddr_pmu\000 */ }, }, }; @@ -211,6 +248,15 @@ struct pmu_events_map { */ const struct pmu_events_map pmu_events_map[] = { { + .arch = "common", + .cpuid = "common", + .event_table = { + .pmus = pmu_events__common, + .num_pmus = ARRAY_SIZE(pmu_events__common), + }, + .metric_table = {}, +}, +{ .arch = "testarch", .cpuid = "testcpu", .event_table = { @@ -380,7 +426,7 @@ int pmu_events_table__for_each_event(const struct pmu_events_table *table, continue; ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data); - if (pmu || ret) + if (ret) return ret; } return 0; @@ -457,11 +503,11 @@ int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, return 0; } -static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) +static const struct pmu_events_map *map_for_cpu(struct perf_cpu cpu) { static struct { const struct pmu_events_map *map; - struct perf_pmu *pmu; + struct perf_cpu cpu; } last_result; static struct { const struct pmu_events_map *map; @@ -472,10 +518,10 @@ static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) char *cpuid = NULL; size_t i; - if (has_last_result && last_result.pmu == pmu) + if (has_last_result && last_result.cpu.cpu == cpu.cpu) return last_result.map; - cpuid = perf_pmu__getcpuid(pmu); + cpuid = get_cpuid_allow_env_override(cpu); /* * On some platforms which uses cpus map, cpuid can be NULL for @@ -506,12 +552,21 @@ static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) has_last_map_search = true; } out_update_last_result: - last_result.pmu = pmu; + last_result.cpu = cpu; last_result.map = map; has_last_result = true; return map; } +static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) +{ + struct perf_cpu cpu = {-1}; + + if (pmu) + cpu = perf_cpu_map__min(pmu->cpus); + return map_for_cpu(cpu); +} + const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) { const struct pmu_events_map *map = map_for_pmu(pmu); @@ -532,24 +587,12 @@ const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) return NULL; } -const struct pmu_metrics_table *perf_pmu__find_metrics_table(struct perf_pmu *pmu) +const struct pmu_metrics_table *pmu_metrics_table__find(void) { - const struct pmu_events_map *map = map_for_pmu(pmu); - - if (!map) - return NULL; + struct perf_cpu cpu = {-1}; + const struct pmu_events_map *map = map_for_cpu(cpu); - if (!pmu) - return &map->metric_table; - - for (size_t i = 0; i < map->metric_table.num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &map->metric_table.pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - - if (pmu__name_match(pmu, pmu_name)) - return &map->metric_table; - } - return NULL; + return map ? &map->metric_table : NULL; } const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid) diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index bb0a5d92df4a..d781a377757a 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -292,6 +292,7 @@ class JsonEvent: 'cpu_atom': 'cpu_atom', 'ali_drw': 'ali_drw', 'arm_cmn': 'arm_cmn', + 'tool': 'tool', } return table[unit] if unit in table else f'uncore_{unit.lower()}' @@ -722,6 +723,17 @@ const struct pmu_events_map pmu_events_map[] = { \t} }, """) + elif arch == 'common': + _args.output_file.write("""{ +\t.arch = "common", +\t.cpuid = "common", +\t.event_table = { +\t\t.pmus = pmu_events__common, +\t\t.num_pmus = ARRAY_SIZE(pmu_events__common), +\t}, +\t.metric_table = {}, +}, +""") else: with open(f'{_args.starting_dir}/{arch}/mapfile.csv') as csvfile: table = csv.reader(csvfile) @@ -930,7 +942,7 @@ int pmu_events_table__for_each_event(const struct pmu_events_table *table, continue; ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data); - if (pmu || ret) + if (ret) return ret; } return 0; @@ -1007,11 +1019,11 @@ int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, return 0; } -static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) +static const struct pmu_events_map *map_for_cpu(struct perf_cpu cpu) { static struct { const struct pmu_events_map *map; - struct perf_pmu *pmu; + struct perf_cpu cpu; } last_result; static struct { const struct pmu_events_map *map; @@ -1022,10 +1034,10 @@ static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) char *cpuid = NULL; size_t i; - if (has_last_result && last_result.pmu == pmu) + if (has_last_result && last_result.cpu.cpu == cpu.cpu) return last_result.map; - cpuid = perf_pmu__getcpuid(pmu); + cpuid = get_cpuid_allow_env_override(cpu); /* * On some platforms which uses cpus map, cpuid can be NULL for @@ -1056,12 +1068,21 @@ static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) has_last_map_search = true; } out_update_last_result: - last_result.pmu = pmu; + last_result.cpu = cpu; last_result.map = map; has_last_result = true; return map; } +static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) +{ + struct perf_cpu cpu = {-1}; + + if (pmu) + cpu = perf_cpu_map__min(pmu->cpus); + return map_for_cpu(cpu); +} + const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) { const struct pmu_events_map *map = map_for_pmu(pmu); @@ -1082,24 +1103,12 @@ const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) return NULL; } -const struct pmu_metrics_table *perf_pmu__find_metrics_table(struct perf_pmu *pmu) +const struct pmu_metrics_table *pmu_metrics_table__find(void) { - const struct pmu_events_map *map = map_for_pmu(pmu); - - if (!map) - return NULL; - - if (!pmu) - return &map->metric_table; + struct perf_cpu cpu = {-1}; + const struct pmu_events_map *map = map_for_cpu(cpu); - for (size_t i = 0; i < map->metric_table.num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &map->metric_table.pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - - if (pmu__name_match(pmu, pmu_name)) - return &map->metric_table; - } - return NULL; + return map ? &map->metric_table : NULL; } const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid) @@ -1241,7 +1250,7 @@ def main() -> None: if len(parents) == _args.model.split(',')[0].count('/'): # We're testing the correct directory. item_path = '/'.join(parents) + ('/' if len(parents) > 0 else '') + item.name - if 'test' not in item_path and item_path not in _args.model.split(','): + if 'test' not in item_path and 'common' not in item_path and item_path not in _args.model.split(','): continue action(parents, item) if item.is_dir(): @@ -1289,7 +1298,7 @@ struct pmu_table_entry { for item in os.scandir(_args.starting_dir): if not item.is_dir(): continue - if item.name == _args.arch or _args.arch == 'all' or item.name == 'test': + if item.name == _args.arch or _args.arch == 'all' or item.name == 'test' or item.name == 'common': archs.append(item.name) if len(archs) < 2 and _args.arch != 'none': diff --git a/tools/perf/pmu-events/pmu-events.h b/tools/perf/pmu-events/pmu-events.h index 5435ad92180c..675562e6f770 100644 --- a/tools/perf/pmu-events/pmu-events.h +++ b/tools/perf/pmu-events/pmu-events.h @@ -103,7 +103,7 @@ int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, pm void *data); const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu); -const struct pmu_metrics_table *perf_pmu__find_metrics_table(struct perf_pmu *pmu); +const struct pmu_metrics_table *pmu_metrics_table__find(void); const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid); const struct pmu_metrics_table *find_core_metrics_table(const char *arch, const char *cpuid); int pmu_for_each_core_event(pmu_event_iter_fn fn, void *data); diff --git a/tools/perf/scripts/python/Perf-Trace-Util/Context.c b/tools/perf/scripts/python/Perf-Trace-Util/Context.c index 3954bd1587ce..01f54d6724a5 100644 --- a/tools/perf/scripts/python/Perf-Trace-Util/Context.c +++ b/tools/perf/scripts/python/Perf-Trace-Util/Context.c @@ -12,6 +12,7 @@ #define PY_SSIZE_T_CLEAN #include <Python.h> +#include "../../../util/config.h" #include "../../../util/trace-event.h" #include "../../../util/event.h" #include "../../../util/symbol.h" @@ -182,6 +183,15 @@ static PyObject *perf_sample_srccode(PyObject *obj, PyObject *args) return perf_sample_src(obj, args, true); } +static PyObject *__perf_config_get(PyObject *obj, PyObject *args) +{ + const char *config_name; + + if (!PyArg_ParseTuple(args, "s", &config_name)) + return NULL; + return Py_BuildValue("s", perf_config_get(config_name)); +} + static PyMethodDef ContextMethods[] = { #ifdef HAVE_LIBTRACEEVENT { "common_pc", perf_trace_context_common_pc, METH_VARARGS, @@ -199,6 +209,7 @@ static PyMethodDef ContextMethods[] = { METH_VARARGS, "Get source file name and line number."}, { "perf_sample_srccode", perf_sample_srccode, METH_VARARGS, "Get source file name, line number and line."}, + { "perf_config_get", __perf_config_get, METH_VARARGS, "Get perf config entry"}, { NULL, NULL, 0, NULL} }; diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index 7aff02d84ffb..ba208c90d631 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -11,36 +11,74 @@ import os from os import path import re from subprocess import * -from optparse import OptionParser, make_option +import argparse +import platform -from perf_trace_context import perf_set_itrace_options, \ - perf_sample_insn, perf_sample_srccode +from perf_trace_context import perf_sample_srccode, perf_config_get # Below are some example commands for using this script. +# Note a --kcore recording is required for accurate decode +# due to the alternatives patching mechanism. However this +# script only supports reading vmlinux for disassembly dump, +# meaning that any patched instructions will appear +# as unpatched, but the instruction ranges themselves will +# be correct. In addition to this, source line info comes +# from Perf, and when using kcore there is no debug info. The +# following lists the supported features in each mode: +# +# +-----------+-----------------+------------------+------------------+ +# | Recording | Accurate decode | Source line dump | Disassembly dump | +# +-----------+-----------------+------------------+------------------+ +# | --kcore | yes | no | yes | +# | normal | no | yes | yes | +# +-----------+-----------------+------------------+------------------+ +# +# Output disassembly with objdump and auto detect vmlinux +# (when running on same machine.) +# perf script -s scripts/python/arm-cs-trace-disasm.py -d # -# Output disassembly with objdump: -# perf script -s scripts/python/arm-cs-trace-disasm.py \ -# -- -d objdump -k path/to/vmlinux # Output disassembly with llvm-objdump: # perf script -s scripts/python/arm-cs-trace-disasm.py \ # -- -d llvm-objdump-11 -k path/to/vmlinux +# # Output only source line and symbols: # perf script -s scripts/python/arm-cs-trace-disasm.py -# Command line parsing. -option_list = [ - # formatting options for the bottom entry of the stack - make_option("-k", "--vmlinux", dest="vmlinux_name", - help="Set path to vmlinux file"), - make_option("-d", "--objdump", dest="objdump_name", - help="Set path to objdump executable file"), - make_option("-v", "--verbose", dest="verbose", - action="store_true", default=False, - help="Enable debugging log") -] +def default_objdump(): + config = perf_config_get("annotate.objdump") + return config if config else "objdump" -parser = OptionParser(option_list=option_list) -(options, args) = parser.parse_args() +# Command line parsing. +def int_arg(v): + v = int(v) + if v < 0: + raise argparse.ArgumentTypeError("Argument must be a positive integer") + return v + +args = argparse.ArgumentParser() +args.add_argument("-k", "--vmlinux", + help="Set path to vmlinux file. Omit to autodetect if running on same machine") +args.add_argument("-d", "--objdump", nargs="?", const=default_objdump(), + help="Show disassembly. Can also be used to change the objdump path"), +args.add_argument("-v", "--verbose", action="store_true", help="Enable debugging log") +args.add_argument("--start-time", type=int_arg, help="Monotonic clock time of sample to start from. " + "See 'time' field on samples in -v mode.") +args.add_argument("--stop-time", type=int_arg, help="Monotonic clock time of sample to stop at. " + "See 'time' field on samples in -v mode.") +args.add_argument("--start-sample", type=int_arg, help="Index of sample to start from. " + "See 'index' field on samples in -v mode.") +args.add_argument("--stop-sample", type=int_arg, help="Index of sample to stop at. " + "See 'index' field on samples in -v mode.") + +options = args.parse_args() +if (options.start_time and options.stop_time and + options.start_time >= options.stop_time): + print("--start-time must less than --stop-time") + exit(2) +if (options.start_sample and options.stop_sample and + options.start_sample >= options.stop_sample): + print("--start-sample must less than --stop-sample") + exit(2) # Initialize global dicts and regular expression disasm_cache = dict() @@ -48,11 +86,23 @@ cpu_data = dict() disasm_re = re.compile(r"^\s*([0-9a-fA-F]+):") disasm_func_re = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") cache_size = 64*1024 +sample_idx = -1 glb_source_file_name = None glb_line_number = None glb_dso = None +kver = platform.release() +vmlinux_paths = [ + f"/usr/lib/debug/boot/vmlinux-{kver}.debug", + f"/usr/lib/debug/lib/modules/{kver}/vmlinux", + f"/lib/modules/{kver}/build/vmlinux", + f"/usr/lib/debug/boot/vmlinux-{kver}", + f"/boot/vmlinux-{kver}", + f"/boot/vmlinux", + f"vmlinux" +] + def get_optional(perf_dict, field): if field in perf_dict: return perf_dict[field] @@ -63,12 +113,25 @@ def get_offset(perf_dict, field): return "+%#x" % perf_dict[field] return "" +def find_vmlinux(): + if hasattr(find_vmlinux, "path"): + return find_vmlinux.path + + for v in vmlinux_paths: + if os.access(v, os.R_OK): + find_vmlinux.path = v + break + else: + find_vmlinux.path = None + + return find_vmlinux.path + def get_dso_file_path(dso_name, dso_build_id): if (dso_name == "[kernel.kallsyms]" or dso_name == "vmlinux"): - if (options.vmlinux_name): - return options.vmlinux_name; + if (options.vmlinux): + return options.vmlinux; else: - return dso_name + return find_vmlinux() if find_vmlinux() else dso_name if (dso_name == "[vdso]") : append = "/vdso" @@ -92,7 +155,7 @@ def read_disam(dso_fname, dso_start, start_addr, stop_addr): else: start_addr = start_addr - dso_start; stop_addr = stop_addr - dso_start; - disasm = [ options.objdump_name, "-d", "-z", + disasm = [ options.objdump, "-d", "-z", "--start-address="+format(start_addr,"#x"), "--stop-address="+format(stop_addr,"#x") ] disasm += [ dso_fname ] @@ -112,10 +175,10 @@ def print_disam(dso_fname, dso_start, start_addr, stop_addr): def print_sample(sample): print("Sample = { cpu: %04d addr: 0x%016x phys_addr: 0x%016x ip: 0x%016x " \ - "pid: %d tid: %d period: %d time: %d }" % \ + "pid: %d tid: %d period: %d time: %d index: %d}" % \ (sample['cpu'], sample['addr'], sample['phys_addr'], \ sample['ip'], sample['pid'], sample['tid'], \ - sample['period'], sample['time'])) + sample['period'], sample['time'], sample_idx)) def trace_begin(): print('ARM CoreSight Trace Data Assembler Dump') @@ -177,6 +240,7 @@ def print_srccode(comm, param_dict, sample, symbol, dso): def process_event(param_dict): global cache_size global options + global sample_idx sample = param_dict["sample"] comm = param_dict["comm"] @@ -187,11 +251,26 @@ def process_event(param_dict): dso_start = get_optional(param_dict, "dso_map_start") dso_end = get_optional(param_dict, "dso_map_end") symbol = get_optional(param_dict, "symbol") + map_pgoff = get_optional(param_dict, "map_pgoff") + # check for valid map offset + if (str(map_pgoff) == '[unknown]'): + map_pgoff = 0 cpu = sample["cpu"] ip = sample["ip"] addr = sample["addr"] + sample_idx += 1 + + if (options.start_time and sample["time"] < options.start_time): + return + if (options.stop_time and sample["time"] > options.stop_time): + exit(0) + if (options.start_sample and sample_idx < options.start_sample): + return + if (options.stop_sample and sample_idx > options.stop_sample): + exit(0) + if (options.verbose == True): print("Event type: %s" % name) print_sample(sample) @@ -243,9 +322,10 @@ def process_event(param_dict): # Record for previous sample packet cpu_data[str(cpu) + 'addr'] = addr - # Handle CS_ETM_TRACE_ON packet if start_addr=0 and stop_addr=4 - if (start_addr == 0 and stop_addr == 4): - print("CPU%d: CS_ETM_TRACE_ON packet is inserted" % cpu) + # Filter out zero start_address. Optionally identify CS_ETM_TRACE_ON packet + if (start_addr == 0): + if ((stop_addr == 4) and (options.verbose == True)): + print("CPU%d: CS_ETM_TRACE_ON packet is inserted" % cpu) return if (start_addr < int(dso_start) or start_addr > int(dso_end)): @@ -256,19 +336,20 @@ def process_event(param_dict): print("Stop address 0x%x is out of range [ 0x%x .. 0x%x ] for dso %s" % (stop_addr, int(dso_start), int(dso_end), dso)) return - if (options.objdump_name != None): + if (options.objdump != None): # It doesn't need to decrease virtual memory offset for disassembly # for kernel dso and executable file dso, so in this case we set # vm_start to zero. if (dso == "[kernel.kallsyms]" or dso_start == 0x400000): dso_vm_start = 0 + map_pgoff = 0 else: dso_vm_start = int(dso_start) dso_fname = get_dso_file_path(dso, dso_bid) if path.exists(dso_fname): - print_disam(dso_fname, dso_vm_start, start_addr, stop_addr) + print_disam(dso_fname, dso_vm_start, start_addr + map_pgoff, stop_addr + map_pgoff) else: - print("Failed to find dso %s for address range [ 0x%x .. 0x%x ]" % (dso, start_addr, stop_addr)) + print("Failed to find dso %s for address range [ 0x%x .. 0x%x ]" % (dso, start_addr + map_pgoff, stop_addr + map_pgoff)) print_srccode(comm, param_dict, sample, symbol, dso) diff --git a/tools/perf/tests/Build b/tools/perf/tests/Build index 5671ee530019..ec4e1f034742 100644 --- a/tools/perf/tests/Build +++ b/tools/perf/tests/Build @@ -4,7 +4,6 @@ perf-test-y += builtin-test.o perf-test-y += tests-scripts.o perf-test-y += parse-events.o perf-test-y += dso-data.o -perf-test-y += attr.o perf-test-y += vmlinux-kallsyms.o perf-test-$(CONFIG_LIBTRACEEVENT) += openat-syscall.o perf-test-$(CONFIG_LIBTRACEEVENT) += openat-syscall-all-cpus.o @@ -67,12 +66,13 @@ perf-test-y += sigtrap.o perf-test-y += event_groups.o perf-test-y += symbols.o perf-test-y += util.o +perf-test-y += hwmon_pmu.o +perf-test-y += tool_pmu.o ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc)) perf-test-$(CONFIG_DWARF_UNWIND) += dwarf-unwind.o endif -CFLAGS_attr.o += -DBINDIR="BUILD_STR($(bindir_SQ))" -DPYTHON="BUILD_STR($(PYTHON_WORD))" CFLAGS_python-use.o += -DPYTHONPATH="BUILD_STR($(OUTPUT)python)" -DPYTHON="BUILD_STR($(PYTHON_WORD))" CFLAGS_dwarf-unwind.o += -fno-optimize-sibling-calls diff --git a/tools/perf/tests/attr.c b/tools/perf/tests/attr.c deleted file mode 100644 index 97e1bdd6ec0e..000000000000 --- a/tools/perf/tests/attr.c +++ /dev/null @@ -1,218 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * The struct perf_event_attr test support. - * - * This test is embedded inside into perf directly and is governed - * by the PERF_TEST_ATTR environment variable and hook inside - * sys_perf_event_open function. - * - * The general idea is to store 'struct perf_event_attr' details for - * each event created within single perf command. Each event details - * are stored into separate text file. Once perf command is finished - * these files can be checked for values we expect for command. - * - * Besides 'struct perf_event_attr' values we also store 'fd' and - * 'group_fd' values to allow checking for groups created. - * - * This all is triggered by setting PERF_TEST_ATTR environment variable. - * It must contain name of existing directory with access and write - * permissions. All the event text files are stored there. - */ - -#include <debug.h> -#include <errno.h> -#include <inttypes.h> -#include <stdlib.h> -#include <stdio.h> -#include <linux/types.h> -#include <linux/kernel.h> -#include <sys/param.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <subcmd/exec-cmd.h> -#include "event.h" -#include "util.h" -#include "tests.h" -#include "pmus.h" - -#define ENV "PERF_TEST_ATTR" - -static char *dir; -static bool ready; - -void test_attr__init(void) -{ - dir = getenv(ENV); - test_attr__enabled = (dir != NULL); -} - -#define BUFSIZE 1024 - -#define __WRITE_ASS(str, fmt, data) \ -do { \ - char buf[BUFSIZE]; \ - size_t size; \ - \ - size = snprintf(buf, BUFSIZE, #str "=%"fmt "\n", data); \ - if (1 != fwrite(buf, size, 1, file)) { \ - perror("test attr - failed to write event file"); \ - fclose(file); \ - return -1; \ - } \ - \ -} while (0) - -#define WRITE_ASS(field, fmt) __WRITE_ASS(field, fmt, attr->field) - -static int store_event(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, - int fd, int group_fd, unsigned long flags) -{ - FILE *file; - char path[PATH_MAX]; - - if (!ready) - return 0; - - snprintf(path, PATH_MAX, "%s/event-%d-%llu-%d", dir, - attr->type, attr->config, fd); - - file = fopen(path, "w+"); - if (!file) { - perror("test attr - failed to open event file"); - return -1; - } - - if (fprintf(file, "[event-%d-%llu-%d]\n", - attr->type, attr->config, fd) < 0) { - perror("test attr - failed to write event file"); - fclose(file); - return -1; - } - - /* syscall arguments */ - __WRITE_ASS(fd, "d", fd); - __WRITE_ASS(group_fd, "d", group_fd); - __WRITE_ASS(cpu, "d", cpu.cpu); - __WRITE_ASS(pid, "d", pid); - __WRITE_ASS(flags, "lu", flags); - - /* struct perf_event_attr */ - WRITE_ASS(type, PRIu32); - WRITE_ASS(size, PRIu32); - WRITE_ASS(config, "llu"); - WRITE_ASS(sample_period, "llu"); - WRITE_ASS(sample_type, "llu"); - WRITE_ASS(read_format, "llu"); - WRITE_ASS(disabled, "d"); - WRITE_ASS(inherit, "d"); - WRITE_ASS(pinned, "d"); - WRITE_ASS(exclusive, "d"); - WRITE_ASS(exclude_user, "d"); - WRITE_ASS(exclude_kernel, "d"); - WRITE_ASS(exclude_hv, "d"); - WRITE_ASS(exclude_idle, "d"); - WRITE_ASS(mmap, "d"); - WRITE_ASS(comm, "d"); - WRITE_ASS(freq, "d"); - WRITE_ASS(inherit_stat, "d"); - WRITE_ASS(enable_on_exec, "d"); - WRITE_ASS(task, "d"); - WRITE_ASS(watermark, "d"); - WRITE_ASS(precise_ip, "d"); - WRITE_ASS(mmap_data, "d"); - WRITE_ASS(sample_id_all, "d"); - WRITE_ASS(exclude_host, "d"); - WRITE_ASS(exclude_guest, "d"); - WRITE_ASS(exclude_callchain_kernel, "d"); - WRITE_ASS(exclude_callchain_user, "d"); - WRITE_ASS(mmap2, "d"); - WRITE_ASS(comm_exec, "d"); - WRITE_ASS(context_switch, "d"); - WRITE_ASS(write_backward, "d"); - WRITE_ASS(namespaces, "d"); - WRITE_ASS(use_clockid, "d"); - WRITE_ASS(wakeup_events, PRIu32); - WRITE_ASS(bp_type, PRIu32); - WRITE_ASS(config1, "llu"); - WRITE_ASS(config2, "llu"); - WRITE_ASS(branch_sample_type, "llu"); - WRITE_ASS(sample_regs_user, "llu"); - WRITE_ASS(sample_stack_user, PRIu32); - - fclose(file); - return 0; -} - -void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, - int fd, int group_fd, unsigned long flags) -{ - int errno_saved = errno; - - if ((fd != -1) && store_event(attr, pid, cpu, fd, group_fd, flags)) { - pr_err("test attr FAILED"); - exit(128); - } - - errno = errno_saved; -} - -void test_attr__ready(void) -{ - if (unlikely(test_attr__enabled) && !ready) - ready = true; -} - -static int run_dir(const char *d, const char *perf) -{ - char v[] = "-vvvvv"; - int vcnt = min(verbose, (int) sizeof(v) - 1); - char cmd[3*PATH_MAX]; - - if (verbose > 0) - vcnt++; - - scnprintf(cmd, 3*PATH_MAX, PYTHON " %s/attr.py -d %s/attr/ -p %s %.*s", - d, d, perf, vcnt, v); - - return system(cmd) ? TEST_FAIL : TEST_OK; -} - -static int test__attr(struct test_suite *test __maybe_unused, int subtest __maybe_unused) -{ - struct stat st; - char path_perf[PATH_MAX]; - char path_dir[PATH_MAX]; - char *exec_path; - - if (perf_pmus__num_core_pmus() > 1) { - /* - * TODO: Attribute tests hard code the PMU type. If there are >1 - * core PMU then each PMU will have a different type which - * requires additional support. - */ - pr_debug("Skip test on hybrid systems"); - return TEST_SKIP; - } - - /* First try development tree tests. */ - if (!lstat("./tests", &st)) - return run_dir("./tests", "./perf"); - - exec_path = get_argv_exec_path(); - if (exec_path == NULL) - return -1; - - /* Then installed path. */ - snprintf(path_dir, PATH_MAX, "%s/tests", exec_path); - snprintf(path_perf, PATH_MAX, "%s/perf", BINDIR); - free(exec_path); - - if (!lstat(path_dir, &st) && - !lstat(path_perf, &st)) - return run_dir(path_dir, path_perf); - - return TEST_SKIP; -} - -DEFINE_SUITE("Setup struct perf_event_attr", attr); diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c index 470a9709427d..8dcf74d3c0a3 100644 --- a/tools/perf/tests/builtin-test.c +++ b/tools/perf/tests/builtin-test.c @@ -8,6 +8,7 @@ #include <errno.h> #include <poll.h> #include <unistd.h> +#include <setjmp.h> #include <string.h> #include <stdlib.h> #include <sys/types.h> @@ -39,11 +40,8 @@ * making them easier to debug. */ static bool dont_fork; -/* Don't fork the tests in parallel and wait for their completion. */ -static bool sequential = true; -/* Do it in parallel, lacks infrastructure to avoid running tests that clash for resources, - * So leave it as the developers choice to enable while working on the needed infra */ -static bool parallel; +/* Fork the tests in parallel and wait for their completion. */ +static bool sequential; const char *dso_to_test; const char *test_objdump_path = "objdump"; @@ -73,13 +71,14 @@ static struct test_suite *generic_tests[] = { &suite__PERF_RECORD, &suite__pmu, &suite__pmu_events, + &suite__hwmon_pmu, + &suite__tool_pmu, &suite__dso_data, &suite__perf_evsel__roundtrip_name_test, #ifdef HAVE_LIBTRACEEVENT &suite__perf_evsel__tp_sched_test, &suite__syscall_openat_tp_fields, #endif - &suite__attr, &suite__hists_link, &suite__python_use, &suite__bp_signal, @@ -139,12 +138,6 @@ static struct test_suite *generic_tests[] = { NULL, }; -static struct test_suite **tests[] = { - generic_tests, - arch_tests, - NULL, /* shell tests created at runtime. */ -}; - static struct test_workload *workloads[] = { &workload__noploop, &workload__thloop, @@ -155,6 +148,9 @@ static struct test_workload *workloads[] = { &workload__landlock, }; +#define workloads__for_each(workload) \ + for (unsigned i = 0; i < ARRAY_SIZE(workloads) && ({ workload = workloads[i]; 1; }); i++) + static int num_subtests(const struct test_suite *t) { int num; @@ -198,6 +194,14 @@ static test_fnptr test_function(const struct test_suite *t, int subtest) return t->test_cases[subtest].run_case; } +static bool test_exclusive(const struct test_suite *t, int subtest) +{ + if (subtest <= 0) + return t->test_cases[0].exclusive; + + return t->test_cases[subtest].exclusive; +} + static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[]) { int i; @@ -229,20 +233,47 @@ struct child_test { int subtest; }; +static jmp_buf run_test_jmp_buf; + +static void child_test_sig_handler(int sig) +{ + siglongjmp(run_test_jmp_buf, sig); +} + static int run_test_child(struct child_process *process) { + const int signals[] = { + SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGINT, SIGPIPE, SIGQUIT, SIGSEGV, SIGTERM, + }; struct child_test *child = container_of(process, struct child_test, process); int err; + err = sigsetjmp(run_test_jmp_buf, 1); + if (err) { + fprintf(stderr, "\n---- unexpected signal (%d) ----\n", err); + err = err > 0 ? -err : -1; + goto err_out; + } + + for (size_t i = 0; i < ARRAY_SIZE(signals); i++) + signal(signals[i], child_test_sig_handler); + pr_debug("--- start ---\n"); pr_debug("test child forked, pid %d\n", getpid()); err = test_function(child->test, child->subtest)(child->test, child->subtest); pr_debug("---- end(%d) ----\n", err); + +err_out: fflush(NULL); + for (size_t i = 0; i < ARRAY_SIZE(signals); i++) + signal(signals[i], SIG_DFL); return -err; } -static int print_test_result(struct test_suite *t, int i, int subtest, int result, int width) +#define TEST_RUNNING -3 + +static int print_test_result(struct test_suite *t, int i, int subtest, int result, int width, + int running) { if (has_subtests(t)) { int subw = width > 2 ? width - 2 : width; @@ -252,6 +283,9 @@ static int print_test_result(struct test_suite *t, int i, int subtest, int resul pr_info("%3d: %-*s:", i + 1, width, test_description(t, subtest)); switch (result) { + case TEST_RUNNING: + color_fprintf(stderr, PERF_COLOR_YELLOW, " Running (%d active)\n", running); + break; case TEST_OK: pr_info(" Ok\n"); break; @@ -273,16 +307,25 @@ static int print_test_result(struct test_suite *t, int i, int subtest, int resul return 0; } -static int finish_test(struct child_test *child_test, int width) +static void finish_test(struct child_test **child_tests, int running_test, int child_test_num, + int width) { - struct test_suite *t = child_test->test; - int i = child_test->test_num; - int subi = child_test->subtest; - int err = child_test->process.err; - bool err_done = err <= 0; + struct child_test *child_test = child_tests[running_test]; + struct test_suite *t; + int i, subi, err; + bool err_done = false; struct strbuf err_output = STRBUF_INIT; + int last_running = -1; int ret; + if (child_test == NULL) { + /* Test wasn't started. */ + return; + } + t = child_test->test; + i = child_test->test_num; + subi = child_test->subtest; + err = child_test->process.err; /* * For test suites with subtests, display the suite name ahead of the * sub test names. @@ -294,7 +337,7 @@ static int finish_test(struct child_test *child_test, int width) * Busy loop reading from the child's stdout/stderr that are set to be * non-blocking until EOF. */ - if (!err_done) + if (err > 0) fcntl(err, F_SETFL, O_NONBLOCK); if (verbose > 1) { if (has_subtests(t)) @@ -308,57 +351,90 @@ static int finish_test(struct child_test *child_test, int width) .events = POLLIN | POLLERR | POLLHUP | POLLNVAL, }, }; - char buf[512]; - ssize_t len; + if (perf_use_color_default) { + int running = 0; - /* Poll to avoid excessive spinning, timeout set for 100ms. */ - poll(pfds, ARRAY_SIZE(pfds), /*timeout=*/100); - if (!err_done && pfds[0].revents) { - errno = 0; - len = read(err, buf, sizeof(buf) - 1); + for (int y = running_test; y < child_test_num; y++) { + if (child_tests[y] == NULL) + continue; + if (check_if_command_finished(&child_tests[y]->process) == 0) + running++; + } + if (running != last_running) { + if (last_running != -1) { + /* + * Erase "Running (.. active)" line + * printed before poll/sleep. + */ + fprintf(debug_file(), PERF_COLOR_DELETE_LINE); + } + print_test_result(t, i, subi, TEST_RUNNING, width, running); + last_running = running; + } + } - if (len <= 0) { - err_done = errno != EAGAIN; - } else { - buf[len] = '\0'; - if (verbose > 1) - fprintf(stdout, "%s", buf); - else + err_done = true; + if (err <= 0) { + /* No child stderr to poll, sleep for 10ms for child to complete. */ + usleep(10 * 1000); + } else { + /* Poll to avoid excessive spinning, timeout set for 100ms. */ + poll(pfds, ARRAY_SIZE(pfds), /*timeout=*/100); + if (pfds[0].revents) { + char buf[512]; + ssize_t len; + + len = read(err, buf, sizeof(buf) - 1); + + if (len > 0) { + err_done = false; + buf[len] = '\0'; strbuf_addstr(&err_output, buf); + } } } + if (err_done) + err_done = check_if_command_finished(&child_test->process); + } + if (perf_use_color_default && last_running != -1) { + /* Erase "Running (.. active)" line printed before poll/sleep. */ + fprintf(debug_file(), PERF_COLOR_DELETE_LINE); } /* Clean up child process. */ ret = finish_command(&child_test->process); - if (verbose == 1 && ret == TEST_FAIL) { - /* Add header for test that was skipped above. */ - if (has_subtests(t)) - pr_info("%3d.%1d: %s:\n", i + 1, subi + 1, test_description(t, subi)); - else - pr_info("%3d: %s:\n", i + 1, test_description(t, -1)); + if (verbose > 1 || (verbose == 1 && ret == TEST_FAIL)) fprintf(stderr, "%s", err_output.buf); - } + strbuf_release(&err_output); - print_test_result(t, i, subi, ret, width); + print_test_result(t, i, subi, ret, width, /*running=*/0); if (err > 0) close(err); - return 0; + zfree(&child_tests[running_test]); } static int start_test(struct test_suite *test, int i, int subi, struct child_test **child, - int width) + int width, int pass) { int err; *child = NULL; if (dont_fork) { - pr_debug("--- start ---\n"); - err = test_function(test, subi)(test, subi); - pr_debug("---- end ----\n"); - print_test_result(test, i, subi, err, width); + if (pass == 1) { + pr_debug("--- start ---\n"); + err = test_function(test, subi)(test, subi); + pr_debug("---- end ----\n"); + print_test_result(test, i, subi, err, width, /*running=*/0); + } + return 0; + } + if (pass == 1 && !sequential && test_exclusive(test, subi)) { + /* When parallel, skip exclusive tests on the first pass. */ + return 0; + } + if (pass != 1 && (sequential || !test_exclusive(test, subi))) { + /* Sequential and non-exclusive tests were run on the first pass. */ return 0; } - *child = zalloc(sizeof(**child)); if (!*child) return -ENOMEM; @@ -377,35 +453,42 @@ static int start_test(struct test_suite *test, int i, int subi, struct child_tes (*child)->process.err = -1; } (*child)->process.no_exec_cmd = run_test_child; - err = start_command(&(*child)->process); - if (err || !sequential) - return err; - return finish_test(*child, width); + if (sequential || pass == 2) { + err = start_command(&(*child)->process); + if (err) + return err; + finish_test(child, /*running_test=*/0, /*child_test_num=*/1, width); + return 0; + } + return start_command(&(*child)->process); } -#define for_each_test(j, k, t) \ - for (j = 0, k = 0; j < ARRAY_SIZE(tests); j++, k = 0) \ - while ((t = tests[j][k++]) != NULL) +/* State outside of __cmd_test for the sake of the signal handler. */ + +static size_t num_tests; +static struct child_test **child_tests; +static jmp_buf cmd_test_jmp_buf; -static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist) +static void cmd_test_sig_handler(int sig) { - struct test_suite *t; - unsigned int j, k; - int i = 0; - int width = 0; - size_t num_tests = 0; - struct child_test **child_tests; - int child_test_num = 0; + siglongjmp(cmd_test_jmp_buf, sig); +} + +static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], + struct intlist *skiplist) +{ + static int width = 0; + int err = 0; - for_each_test(j, k, t) { - int len = strlen(test_description(t, -1)); + for (struct test_suite **t = suites; *t; t++) { + int len = strlen(test_description(*t, -1)); if (width < len) width = len; - if (has_subtests(t)) { - for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { - len = strlen(test_description(t, subi)); + if (has_subtests(*t)) { + for (int subi = 0, subn = num_subtests(*t); subi < subn; subi++) { + len = strlen(test_description(*t, subi)); if (width < len) width = len; num_tests++; @@ -418,97 +501,137 @@ static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist) if (!child_tests) return -ENOMEM; - for_each_test(j, k, t) { - int curr = i++; - - if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) { - bool skip = true; + err = sigsetjmp(cmd_test_jmp_buf, 1); + if (err) { + pr_err("\nSignal (%d) while running tests.\nTerminating tests with the same signal\n", + err); + for (size_t x = 0; x < num_tests; x++) { + struct child_test *child_test = child_tests[x]; - for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { - if (perf_test__matches(test_description(t, subi), - curr, argc, argv)) - skip = false; - } - - if (skip) + if (!child_test) continue; - } - if (intlist__find(skiplist, i)) { - pr_info("%3d: %-*s:", curr + 1, width, test_description(t, -1)); - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n"); - continue; + pr_debug3("Killing %d pid %d\n", + child_test->test_num + 1, + child_test->process.pid); + kill(child_test->process.pid, err); } + goto err_out; + } + signal(SIGINT, cmd_test_sig_handler); + signal(SIGTERM, cmd_test_sig_handler); - if (!has_subtests(t)) { - int err = start_test(t, curr, -1, &child_tests[child_test_num++], width); + /* + * In parallel mode pass 1 runs non-exclusive tests in parallel, pass 2 + * runs the exclusive tests sequentially. In other modes all tests are + * run in pass 1. + */ + for (int pass = 1; pass <= 2; pass++) { + int child_test_num = 0; + int i = 0; + + for (struct test_suite **t = suites; *t; t++) { + int curr = i++; + + if (!perf_test__matches(test_description(*t, -1), curr, argc, argv)) { + /* + * Test suite shouldn't be run based on + * description. See if subtest should. + */ + bool skip = true; + + for (int subi = 0, subn = num_subtests(*t); subi < subn; subi++) { + if (perf_test__matches(test_description(*t, subi), + curr, argc, argv)) + skip = false; + } + + if (skip) + continue; + } - if (err) { - /* TODO: if !sequential waitpid the already forked children. */ - free(child_tests); - return err; + if (intlist__find(skiplist, i)) { + pr_info("%3d: %-*s:", curr + 1, width, test_description(*t, -1)); + color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n"); + continue; } - } else { - for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { - int err; - if (!perf_test__matches(test_description(t, subi), + if (!has_subtests(*t)) { + err = start_test(*t, curr, -1, &child_tests[child_test_num++], + width, pass); + if (err) + goto err_out; + continue; + } + for (int subi = 0, subn = num_subtests(*t); subi < subn; subi++) { + if (!perf_test__matches(test_description(*t, subi), curr, argc, argv)) continue; - err = start_test(t, curr, subi, &child_tests[child_test_num++], - width); + err = start_test(*t, curr, subi, &child_tests[child_test_num++], + width, pass); if (err) - return err; + goto err_out; } } - } - for (i = 0; i < child_test_num; i++) { if (!sequential) { - int ret = finish_test(child_tests[i], width); - - if (ret) - return ret; + /* Parallel mode starts tests but doesn't finish them. Do that now. */ + for (size_t x = 0; x < num_tests; x++) + finish_test(child_tests, x, num_tests, width); } - free(child_tests[i]); + } +err_out: + signal(SIGINT, SIG_DFL); + signal(SIGTERM, SIG_DFL); + if (err) { + pr_err("Internal test harness failure. Completing any started tests:\n:"); + for (size_t x = 0; x < num_tests; x++) + finish_test(child_tests, x, num_tests, width); } free(child_tests); - return 0; + return err; } -static int perf_test__list(int argc, const char **argv) +static int perf_test__list(struct test_suite **suites, int argc, const char **argv) { - unsigned int j, k; - struct test_suite *t; int i = 0; - for_each_test(j, k, t) { + for (struct test_suite **t = suites; *t; t++) { int curr = i++; - if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) + if (!perf_test__matches(test_description(*t, -1), curr, argc, argv)) continue; - pr_info("%3d: %s\n", i, test_description(t, -1)); + pr_info("%3d: %s\n", i, test_description(*t, -1)); - if (has_subtests(t)) { - int subn = num_subtests(t); + if (has_subtests(*t)) { + int subn = num_subtests(*t); int subi; for (subi = 0; subi < subn; subi++) pr_info("%3d:%1d: %s\n", i, subi + 1, - test_description(t, subi)); + test_description(*t, subi)); } } return 0; } +static int workloads__fprintf_list(FILE *fp) +{ + struct test_workload *twl; + int printed = 0; + + workloads__for_each(twl) + printed += fprintf(fp, "%s\n", twl->name); + + return printed; +} + static int run_workload(const char *work, int argc, const char **argv) { - unsigned int i = 0; struct test_workload *twl; - for (i = 0; i < ARRAY_SIZE(workloads); i++) { - twl = workloads[i]; + workloads__for_each(twl) { if (!strcmp(twl->name, work)) return twl->func(argc, argv); } @@ -526,6 +649,55 @@ static int perf_test__config(const char *var, const char *value, return 0; } +static struct test_suite **build_suites(void) +{ + /* + * TODO: suites is static to avoid needing to clean up the scripts tests + * for leak sanitizer. + */ + static struct test_suite **suites[] = { + generic_tests, + arch_tests, + NULL, + }; + struct test_suite **result; + struct test_suite *t; + size_t n = 0, num_suites = 0; + + if (suites[2] == NULL) + suites[2] = create_script_test_suites(); + +#define for_each_test(t) \ + for (size_t i = 0, j = 0; i < ARRAY_SIZE(suites); i++, j = 0) \ + while ((t = suites[i][j++]) != NULL) + + for_each_test(t) + num_suites++; + + result = calloc(num_suites + 1, sizeof(struct test_suite *)); + + for (int pass = 1; pass <= 2; pass++) { + for_each_test(t) { + bool exclusive = false; + + if (!has_subtests(t)) { + exclusive = test_exclusive(t, -1); + } else { + for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) { + if (test_exclusive(t, subi)) { + exclusive = true; + break; + } + } + } + if ((!exclusive && pass == 1) || (exclusive && pass == 2)) + result[n++] = t; + } + } + return result; +#undef for_each_test +} + int cmd_test(int argc, const char **argv) { const char *test_usage[] = { @@ -534,16 +706,17 @@ int cmd_test(int argc, const char **argv) }; const char *skip = NULL; const char *workload = NULL; + bool list_workloads = false; const struct option test_options[] = { OPT_STRING('s', "skip", &skip, "tests", "tests to skip"), OPT_INCR('v', "verbose", &verbose, "be more verbose (show symbol address, etc)"), OPT_BOOLEAN('F', "dont-fork", &dont_fork, "Do not fork for testcase"), - OPT_BOOLEAN('p', "parallel", ¶llel, "Run the tests in parallel"), OPT_BOOLEAN('S', "sequential", &sequential, "Run the tests one after another rather than in parallel"), - OPT_STRING('w', "workload", &workload, "work", "workload to run for testing"), + OPT_STRING('w', "workload", &workload, "work", "workload to run for testing, use '--list-workloads' to list the available ones."), + OPT_BOOLEAN(0, "list-workloads", &list_workloads, "List the available builtin workloads to use with -w/--workload"), OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"), OPT_STRING(0, "objdump", &test_objdump_path, "path", "objdump binary to use for disassembly and annotations"), @@ -552,6 +725,7 @@ int cmd_test(int argc, const char **argv) const char * const test_subcommands[] = { "list", NULL }; struct intlist *skiplist = NULL; int ret = hists__init(); + struct test_suite **suites; if (ret < 0) return ret; @@ -561,22 +735,29 @@ int cmd_test(int argc, const char **argv) /* Unbuffered output */ setvbuf(stdout, NULL, _IONBF, 0); - tests[2] = create_script_test_suites(); argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0); - if (argc >= 1 && !strcmp(argv[0], "list")) - return perf_test__list(argc - 1, argv + 1); + if (argc >= 1 && !strcmp(argv[0], "list")) { + suites = build_suites(); + ret = perf_test__list(suites, argc - 1, argv + 1); + free(suites); + return ret; + } if (workload) return run_workload(workload, argc, argv); + if (list_workloads) { + workloads__fprintf_list(stdout); + return 0; + } + if (dont_fork) sequential = true; - else if (parallel) - sequential = false; symbol_conf.priv_size = sizeof(int); symbol_conf.try_vmlinux_path = true; + if (symbol__init(NULL) < 0) return -1; @@ -588,5 +769,8 @@ int cmd_test(int argc, const char **argv) */ rlimit__bump_memlock(); - return __cmd_test(argc, argv, skiplist); + suites = build_suites(); + ret = __cmd_test(suites, argc, argv, skiplist); + free(suites); + return ret; } diff --git a/tools/perf/tests/demangle-java-test.c b/tools/perf/tests/demangle-java-test.c index 44d1be303b67..93c94408bdc8 100644 --- a/tools/perf/tests/demangle-java-test.c +++ b/tools/perf/tests/demangle-java-test.c @@ -2,6 +2,7 @@ #include <string.h> #include <stdlib.h> #include <stdio.h> +#include <linux/kernel.h> #include "tests.h" #include "session.h" #include "debug.h" @@ -28,7 +29,7 @@ static int test__demangle_java(struct test_suite *test __maybe_unused, int subte "void java.lang.Object<init>()" }, }; - for (i = 0; i < sizeof(test_cases) / sizeof(test_cases[0]); i++) { + for (i = 0; i < ARRAY_SIZE(test_cases); i++) { buf = java_demangle_sym(test_cases[i].mangled, 0); if (strcmp(buf, test_cases[i].demangled)) { pr_debug("FAILED: %s: %s != %s\n", test_cases[i].mangled, diff --git a/tools/perf/tests/event-times.c b/tools/perf/tests/event-times.c index e155f0e0e04d..deefe5003bfc 100644 --- a/tools/perf/tests/event-times.c +++ b/tools/perf/tests/event-times.c @@ -126,6 +126,7 @@ static int attach__cpu_disabled(struct evlist *evlist) evsel->core.attr.disabled = 1; err = evsel__open_per_cpu(evsel, cpus, -1); + perf_cpu_map__put(cpus); if (err) { if (err == -EACCES) return TEST_SKIP; @@ -134,7 +135,6 @@ static int attach__cpu_disabled(struct evlist *evlist) return err; } - perf_cpu_map__put(cpus); return evsel__enable(evsel); } @@ -153,10 +153,10 @@ static int attach__cpu_enabled(struct evlist *evlist) } err = evsel__open_per_cpu(evsel, cpus, -1); + perf_cpu_map__put(cpus); if (err == -EACCES) return TEST_SKIP; - perf_cpu_map__put(cpus); return err ? TEST_FAIL : TEST_OK; } @@ -188,6 +188,7 @@ static int test_times(int (attach)(struct evlist *), err = attach(evlist); if (err == TEST_SKIP) { pr_debug(" SKIP : not enough rights\n"); + evlist__delete(evlist); return err; } diff --git a/tools/perf/tests/evsel-tp-sched.c b/tools/perf/tests/evsel-tp-sched.c index cf4da3d748c2..226196fb9677 100644 --- a/tools/perf/tests/evsel-tp-sched.c +++ b/tools/perf/tests/evsel-tp-sched.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/err.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> #include "evsel.h" #include "tests.h" #include "debug.h" @@ -36,33 +36,33 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse int subtest __maybe_unused) { struct evsel *evsel = evsel__newtp("sched", "sched_switch"); - int ret = 0; + int ret = TEST_OK; if (IS_ERR(evsel)) { pr_debug("evsel__newtp failed with %ld\n", PTR_ERR(evsel)); - return -1; + return PTR_ERR(evsel) == -EACCES ? TEST_SKIP : TEST_FAIL; } if (evsel__test_field(evsel, "prev_comm", 16, false)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "prev_pid", 4, true)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "prev_prio", 4, true)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "prev_state", sizeof(long), true)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "next_comm", 16, false)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "next_pid", 4, true)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "next_prio", 4, true)) - ret = -1; + ret = TEST_FAIL; evsel__delete(evsel); @@ -70,23 +70,33 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (IS_ERR(evsel)) { pr_debug("evsel__newtp failed with %ld\n", PTR_ERR(evsel)); - return -1; + return TEST_FAIL; } if (evsel__test_field(evsel, "comm", 16, false)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "pid", 4, true)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "prio", 4, true)) - ret = -1; + ret = TEST_FAIL; if (evsel__test_field(evsel, "target_cpu", 4, true)) - ret = -1; + ret = TEST_FAIL; evsel__delete(evsel); return ret; } -DEFINE_SUITE("Parse sched tracepoints fields", perf_evsel__tp_sched_test); +static struct test_case tests__perf_evsel__tp_sched_test[] = { + TEST_CASE_REASON("Parse sched tracepoints fields", + perf_evsel__tp_sched_test, + "permissions"), + { .name = NULL, } +}; + +struct test_suite suite__perf_evsel__tp_sched_test = { + .desc = "Parse sched tracepoints fields", + .test_cases = tests__perf_evsel__tp_sched_test, +}; diff --git a/tools/perf/tests/expr.c b/tools/perf/tests/expr.c index e3aa9d4fcf3a..41ff1affdfcd 100644 --- a/tools/perf/tests/expr.c +++ b/tools/perf/tests/expr.c @@ -6,6 +6,7 @@ #include "util/header.h" #include "util/smt.h" #include "tests.h" +#include <perf/cpumap.h> #include <math.h> #include <stdlib.h> #include <string.h> @@ -76,8 +77,8 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u struct expr_parse_ctx *ctx; bool is_intel = false; char strcmp_cpuid_buf[256]; - struct perf_pmu *pmu = perf_pmus__find_core_pmu(); - char *cpuid = perf_pmu__getcpuid(pmu); + struct perf_cpu cpu = {-1}; + char *cpuid = get_cpuid_allow_env_override(cpu); char *escaped_cpuid1, *escaped_cpuid2; TEST_ASSERT_VAL("get_cpuid", cpuid); diff --git a/tools/perf/tests/hwmon_pmu.c b/tools/perf/tests/hwmon_pmu.c new file mode 100644 index 000000000000..f8bcee9660d5 --- /dev/null +++ b/tools/perf/tests/hwmon_pmu.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "debug.h" +#include "evlist.h" +#include "hwmon_pmu.h" +#include "parse-events.h" +#include "tests.h" +#include <fcntl.h> +#include <sys/stat.h> +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/string.h> + +static const struct test_event { + const char *name; + const char *alias; + long config; +} test_events[] = { + { + "temp_test_hwmon_event1", + "temp1", + 0xA0001, + }, + { + "temp_test_hwmon_event2", + "temp2", + 0xA0002, + }, +}; + +/* Cleanup test PMU directory. */ +static int test_pmu_put(const char *dir, struct perf_pmu *hwm) +{ + char buf[PATH_MAX + 20]; + int ret; + + if (scnprintf(buf, sizeof(buf), "rm -fr %s", dir) < 0) { + pr_err("Failure to set up buffer for \"%s\"\n", dir); + return -EINVAL; + } + ret = system(buf); + if (ret) + pr_err("Failure to \"%s\"\n", buf); + + list_del(&hwm->list); + perf_pmu__delete(hwm); + return ret; +} + +/* + * Prepare test PMU directory data, normally exported by kernel at + * /sys/class/hwmon/hwmon<number>/. Give as input a buffer to hold the file + * path, the result is PMU loaded using that directory. + */ +static struct perf_pmu *test_pmu_get(char *dir, size_t sz) +{ + const char *test_hwmon_name_nl = "A test hwmon PMU\n"; + const char *test_hwmon_name = "A test hwmon PMU"; + /* Simulated hwmon items. */ + const struct test_item { + const char *name; + const char *value; + } test_items[] = { + { "temp1_label", "test hwmon event1\n", }, + { "temp1_input", "40000\n", }, + { "temp2_label", "test hwmon event2\n", }, + { "temp2_input", "50000\n", }, + }; + int dirfd, file; + struct perf_pmu *hwm = NULL; + ssize_t len; + + /* Create equivalent of sysfs mount point. */ + scnprintf(dir, sz, "/tmp/perf-hwmon-pmu-test-XXXXXX"); + if (!mkdtemp(dir)) { + pr_err("mkdtemp failed\n"); + dir[0] = '\0'; + return NULL; + } + dirfd = open(dir, O_DIRECTORY); + if (dirfd < 0) { + pr_err("Failed to open test directory \"%s\"\n", dir); + goto err_out; + } + + /* Create the test hwmon directory and give it a name. */ + if (mkdirat(dirfd, "hwmon1234", 0755) < 0) { + pr_err("Failed to mkdir hwmon directory\n"); + goto err_out; + } + file = openat(dirfd, "hwmon1234/name", O_WRONLY | O_CREAT, 0600); + if (!file) { + pr_err("Failed to open for writing file \"name\"\n"); + goto err_out; + } + len = strlen(test_hwmon_name_nl); + if (write(file, test_hwmon_name_nl, len) < len) { + close(file); + pr_err("Failed to write to 'name' file\n"); + goto err_out; + } + close(file); + + /* Create test hwmon files. */ + for (size_t i = 0; i < ARRAY_SIZE(test_items); i++) { + const struct test_item *item = &test_items[i]; + + file = openat(dirfd, item->name, O_WRONLY | O_CREAT, 0600); + if (!file) { + pr_err("Failed to open for writing file \"%s\"\n", item->name); + goto err_out; + } + + if (write(file, item->value, strlen(item->value)) < 0) { + pr_err("Failed to write to file \"%s\"\n", item->name); + close(file); + goto err_out; + } + close(file); + } + + /* Make the PMU reading the files created above. */ + hwm = perf_pmus__add_test_hwmon_pmu(dirfd, "hwmon1234", test_hwmon_name); + if (!hwm) + pr_err("Test hwmon creation failed\n"); + +err_out: + if (!hwm) { + test_pmu_put(dir, hwm); + if (dirfd >= 0) + close(dirfd); + } + return hwm; +} + +static int do_test(size_t i, bool with_pmu, bool with_alias) +{ + const char *test_event = with_alias ? test_events[i].alias : test_events[i].name; + struct evlist *evlist = evlist__new(); + struct evsel *evsel; + struct parse_events_error err; + int ret; + char str[128]; + bool found = false; + + if (!evlist) { + pr_err("evlist allocation failed\n"); + return TEST_FAIL; + } + + if (with_pmu) + snprintf(str, sizeof(str), "hwmon_a_test_hwmon_pmu/%s/", test_event); + else + strlcpy(str, test_event, sizeof(str)); + + pr_debug("Testing '%s'\n", str); + parse_events_error__init(&err); + ret = parse_events(evlist, str, &err); + if (ret) { + pr_debug("FAILED %s:%d failed to parse event '%s', err %d\n", + __FILE__, __LINE__, str, ret); + parse_events_error__print(&err, str); + ret = TEST_FAIL; + goto out; + } + + ret = TEST_OK; + if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", + __FILE__, __LINE__, str, evlist->core.nr_entries); + ret = TEST_FAIL; + goto out; + } + + evlist__for_each_entry(evlist, evsel) { + if (!evsel->pmu || !evsel->pmu->name || + strcmp(evsel->pmu->name, "hwmon_a_test_hwmon_pmu")) + continue; + + if (evsel->core.attr.config != (u64)test_events[i].config) { + pr_debug("FAILED %s:%d Unexpected config for '%s', %lld != %ld\n", + __FILE__, __LINE__, str, + evsel->core.attr.config, + test_events[i].config); + ret = TEST_FAIL; + goto out; + } + found = true; + } + + if (!found) { + pr_debug("FAILED %s:%d Didn't find hwmon event '%s' in parsed evsels\n", + __FILE__, __LINE__, str); + ret = TEST_FAIL; + } + +out: + parse_events_error__exit(&err); + evlist__delete(evlist); + return ret; +} + +static int test__hwmon_pmu(bool with_pmu) +{ + char dir[PATH_MAX]; + struct perf_pmu *pmu = test_pmu_get(dir, sizeof(dir)); + int ret = TEST_OK; + + if (!pmu) + return TEST_FAIL; + + for (size_t i = 0; i < ARRAY_SIZE(test_events); i++) { + ret = do_test(i, with_pmu, /*with_alias=*/false); + + if (ret != TEST_OK) + break; + + ret = do_test(i, with_pmu, /*with_alias=*/true); + + if (ret != TEST_OK) + break; + } + test_pmu_put(dir, pmu); + return ret; +} + +static int test__hwmon_pmu_without_pmu(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + return test__hwmon_pmu(/*with_pmu=*/false); +} + +static int test__hwmon_pmu_with_pmu(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + return test__hwmon_pmu(/*with_pmu=*/true); +} + +static int test__parse_hwmon_filename(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + const struct hwmon_parse_test { + const char *filename; + enum hwmon_type type; + int number; + enum hwmon_item item; + bool alarm; + bool parse_ok; + } tests[] = { + { + .filename = "cpu0_accuracy", + .type = HWMON_TYPE_CPU, + .number = 0, + .item = HWMON_ITEM_ACCURACY, + .alarm = false, + .parse_ok = true, + }, + { + .filename = "temp1_input", + .type = HWMON_TYPE_TEMP, + .number = 1, + .item = HWMON_ITEM_INPUT, + .alarm = false, + .parse_ok = true, + }, + { + .filename = "fan2_vid", + .type = HWMON_TYPE_FAN, + .number = 2, + .item = HWMON_ITEM_VID, + .alarm = false, + .parse_ok = true, + }, + { + .filename = "power3_crit_alarm", + .type = HWMON_TYPE_POWER, + .number = 3, + .item = HWMON_ITEM_CRIT, + .alarm = true, + .parse_ok = true, + }, + { + .filename = "intrusion4_average_interval_min_alarm", + .type = HWMON_TYPE_INTRUSION, + .number = 4, + .item = HWMON_ITEM_AVERAGE_INTERVAL_MIN, + .alarm = true, + .parse_ok = true, + }, + { + .filename = "badtype5_baditem", + .type = HWMON_TYPE_NONE, + .number = 5, + .item = HWMON_ITEM_NONE, + .alarm = false, + .parse_ok = false, + }, + { + .filename = "humidity6_baditem", + .type = HWMON_TYPE_NONE, + .number = 6, + .item = HWMON_ITEM_NONE, + .alarm = false, + .parse_ok = false, + }, + }; + + for (size_t i = 0; i < ARRAY_SIZE(tests); i++) { + enum hwmon_type type; + int number; + enum hwmon_item item; + bool alarm; + + TEST_ASSERT_EQUAL("parse_hwmon_filename", + parse_hwmon_filename( + tests[i].filename, + &type, + &number, + &item, + &alarm), + tests[i].parse_ok + ); + if (tests[i].parse_ok) { + TEST_ASSERT_EQUAL("parse_hwmon_filename type", type, tests[i].type); + TEST_ASSERT_EQUAL("parse_hwmon_filename number", number, tests[i].number); + TEST_ASSERT_EQUAL("parse_hwmon_filename item", item, tests[i].item); + TEST_ASSERT_EQUAL("parse_hwmon_filename alarm", alarm, tests[i].alarm); + } + } + return TEST_OK; +} + +static struct test_case tests__hwmon_pmu[] = { + TEST_CASE("Basic parsing test", parse_hwmon_filename), + TEST_CASE("Parsing without PMU name", hwmon_pmu_without_pmu), + TEST_CASE("Parsing with PMU name", hwmon_pmu_with_pmu), + { .name = NULL, } +}; + +struct test_suite suite__hwmon_pmu = { + .desc = "Hwmon PMU", + .test_cases = tests__hwmon_pmu, +}; diff --git a/tools/perf/tests/make b/tools/perf/tests/make index a5040772043f..a7fcbd589752 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -81,7 +81,7 @@ make_no_gtk2 := NO_GTK2=1 make_no_ui := NO_SLANG=1 NO_GTK2=1 make_no_demangle := NO_DEMANGLE=1 make_no_libelf := NO_LIBELF=1 -make_no_libunwind := NO_LIBUNWIND=1 +make_libunwind := LIBUNWIND=1 make_no_libdw_dwarf_unwind := NO_LIBDW_DWARF_UNWIND=1 make_no_backtrace := NO_BACKTRACE=1 make_no_libcapstone := NO_CAPSTONE=1 @@ -121,7 +121,7 @@ make_static := LDFLAGS=-static NO_PERF_READ_VDSO32=1 NO_PERF_READ_VDSOX3 # all the NO_* variable combined make_minimal := NO_LIBPERL=1 NO_LIBPYTHON=1 NO_GTK2=1 -make_minimal += NO_DEMANGLE=1 NO_LIBELF=1 NO_LIBUNWIND=1 NO_BACKTRACE=1 +make_minimal += NO_DEMANGLE=1 NO_LIBELF=1 NO_BACKTRACE=1 make_minimal += NO_LIBNUMA=1 NO_LIBAUDIT=1 NO_LIBBIONIC=1 make_minimal += NO_LIBDW_DWARF_UNWIND=1 NO_AUXTRACE=1 NO_LIBBPF=1 make_minimal += NO_LIBCRYPTO=1 NO_SDT=1 NO_JVMTI=1 NO_LIBZSTD=1 @@ -153,7 +153,7 @@ run += make_no_gtk2 run += make_no_ui run += make_no_demangle run += make_no_libelf -run += make_no_libunwind +run += make_libunwind run += make_no_libdw_dwarf_unwind run += make_no_backtrace run += make_no_libcapstone diff --git a/tools/perf/tests/openat-syscall-tp-fields.c b/tools/perf/tests/openat-syscall-tp-fields.c index 888df8eca981..3943da441979 100644 --- a/tools/perf/tests/openat-syscall-tp-fields.c +++ b/tools/perf/tests/openat-syscall-tp-fields.c @@ -40,7 +40,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused int flags = O_RDONLY | O_DIRECTORY; struct evlist *evlist = evlist__new(); struct evsel *evsel; - int err = -1, i, nr_events = 0, nr_polls = 0; + int ret = TEST_FAIL, err, i, nr_events = 0, nr_polls = 0; char sbuf[STRERR_BUFSIZE]; if (evlist == NULL) { @@ -51,6 +51,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused evsel = evsel__newtp("syscalls", "sys_enter_openat"); if (IS_ERR(evsel)) { pr_debug("%s: evsel__newtp\n", __func__); + ret = PTR_ERR(evsel) == -EACCES ? TEST_SKIP : TEST_FAIL; goto out_delete_evlist; } @@ -138,11 +139,21 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused } } out_ok: - err = 0; + ret = TEST_OK; out_delete_evlist: evlist__delete(evlist); out: - return err; + return ret; } -DEFINE_SUITE("syscalls:sys_enter_openat event fields", syscall_openat_tp_fields); +static struct test_case tests__syscall_openat_tp_fields[] = { + TEST_CASE_REASON("syscalls:sys_enter_openat event fields", + syscall_openat_tp_fields, + "permissions"), + { .name = NULL, } +}; + +struct test_suite suite__syscall_openat_tp_fields = { + .desc = "syscalls:sys_enter_openat event fields", + .test_cases = tests__syscall_openat_tp_fields, +}; diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index 9e3086d02150..82a19674a38f 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -730,7 +730,7 @@ static int test__checkevent_pmu_events(struct evlist *evlist) TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type || - strcmp(evsel->pmu_name, "cpu")); + strcmp(evsel->pmu->name, "cpu")); TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", @@ -898,8 +898,7 @@ static int test__group1(struct evlist *evlist) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - /* use of precise requires exclude_guest */ - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", evsel->core.attr.precise_ip == 2); TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); @@ -932,7 +931,7 @@ static int test__group2(struct evlist *evlist) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong leader", evsel__is_group_leader(evsel)); @@ -947,7 +946,7 @@ static int test__group2(struct evlist *evlist) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); if (evsel__has_leader(evsel, leader)) @@ -1016,9 +1015,8 @@ static int test__group3(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - /* use of precise requires exclude_guest */ TEST_ASSERT_VAL("wrong exclude guest", - evsel->core.attr.exclude_guest); + !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", @@ -1072,7 +1070,7 @@ static int test__group3(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong leader", evsel__is_group_leader(evsel)); @@ -1103,8 +1101,7 @@ static int test__group4(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - /* use of precise requires exclude_guest */ - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", evsel->core.attr.precise_ip == 1); TEST_ASSERT_VAL("wrong group name", !evsel->group_name); @@ -1122,8 +1119,7 @@ static int test__group4(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_user", evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - /* use of precise requires exclude_guest */ - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", evsel->core.attr.precise_ip == 2); TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); @@ -1222,7 +1218,7 @@ static int test__group5(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", !evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong leader", evsel__is_group_leader(evsel)); @@ -1437,7 +1433,7 @@ static int test__leader_sample1(struct evlist *evlist) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", !evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong group name", !evsel->group_name); @@ -1453,7 +1449,7 @@ static int test__leader_sample1(struct evlist *evlist) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", !evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); @@ -1468,7 +1464,7 @@ static int test__leader_sample1(struct evlist *evlist) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", !evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong group name", !evsel->group_name); @@ -1497,7 +1493,7 @@ static int test__leader_sample2(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong group name", !evsel->group_name); @@ -1513,7 +1509,7 @@ static int test__leader_sample2(struct evlist *evlist __maybe_unused) TEST_ASSERT_VAL("wrong exclude_user", !evsel->core.attr.exclude_user); TEST_ASSERT_VAL("wrong exclude_kernel", evsel->core.attr.exclude_kernel); TEST_ASSERT_VAL("wrong exclude_hv", evsel->core.attr.exclude_hv); - TEST_ASSERT_VAL("wrong exclude guest", evsel->core.attr.exclude_guest); + TEST_ASSERT_VAL("wrong exclude guest", !evsel->core.attr.exclude_guest); TEST_ASSERT_VAL("wrong exclude host", !evsel->core.attr.exclude_host); TEST_ASSERT_VAL("wrong precise_ip", !evsel->core.attr.precise_ip); TEST_ASSERT_VAL("wrong group name", !evsel->group_name); diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index be18506f6a24..6a681e3fb552 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -176,7 +176,8 @@ static int test__pmu_format(struct test_suite *test __maybe_unused, int subtest } memset(&attr, 0, sizeof(attr)); - ret = perf_pmu__config_terms(pmu, &attr, &terms, /*zero=*/false, /*err=*/NULL); + ret = perf_pmu__config_terms(pmu, &attr, &terms, /*zero=*/false, + /*apply_hardcoded=*/false, /*err=*/NULL); if (ret) { pr_err("perf_pmu__config_terms failed"); goto err_out; diff --git a/tools/perf/tests/shell/annotate.sh b/tools/perf/tests/shell/annotate.sh index 2ccf4f1d46b6..1590a37363de 100755 --- a/tools/perf/tests/shell/annotate.sh +++ b/tools/perf/tests/shell/annotate.sh @@ -44,7 +44,7 @@ test_basic() { fi # Generate the annotated output file - perf annotate -i "${perfdata}" --stdio 2> /dev/null > "${perfout}" + perf annotate --no-demangle -i "${perfdata}" --stdio 2> /dev/null | head -250 > "${perfout}" # check if it has the target symbol if ! grep "${testsym}" "${perfout}" @@ -63,8 +63,8 @@ test_basic() { fi # check again with a target symbol name - if ! perf annotate -i "${perfdata}" "${testsym}" 2> /dev/null | \ - grep -m 3 "${disasm_regex}" + if ! perf annotate --no-demangle -i "${perfdata}" "${testsym}" 2> /dev/null | \ + head -250 | grep -m 3 "${disasm_regex}" then echo "Basic annotate [Failed: missing disasm output when specifying the target symbol]" err=1 @@ -72,8 +72,8 @@ test_basic() { fi # check one more with external objdump tool (forced by --objdump option) - if ! perf annotate -i "${perfdata}" --objdump=objdump 2> /dev/null | \ - grep -m 3 "${disasm_regex}" + if ! perf annotate --no-demangle -i "${perfdata}" --objdump=objdump 2> /dev/null | \ + head -250 | grep -m 3 "${disasm_regex}" then echo "Basic annotate [Failed: missing disasm output from non default disassembler (using --objdump)]" err=1 diff --git a/tools/perf/tests/shell/attr.sh b/tools/perf/tests/shell/attr.sh new file mode 100755 index 000000000000..5a4e43b2471d --- /dev/null +++ b/tools/perf/tests/shell/attr.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Perf attribute expectations test +# SPDX-License-Identifier: GPL-2.0 + +err=0 + +cleanup() { + trap - EXIT TERM INT +} + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +shelldir=$(dirname "$0") +perf_path=$(which perf) +python "${shelldir}"/lib/attr.py -d "${shelldir}"/attr -v -p "$perf_path" +cleanup +exit $err diff --git a/tools/perf/tests/attr/README b/tools/perf/tests/shell/attr/README index 4066fec7180a..67c4ca76b85d 100644 --- a/tools/perf/tests/attr/README +++ b/tools/perf/tests/shell/attr/README @@ -51,6 +51,8 @@ Following tests are defined (with perf commands): perf record --call-graph fp kill (test-record-graph-fp-aarch64) perf record -e '{cycles,instructions}' kill (test-record-group1) perf record -e '{cycles/period=1/,instructions/period=2/}:S' kill (test-record-group2) + perf record -e '{cycles,cache-misses}:S' kill (test-record-group-sampling1) + perf record -c 10000 -e '{cycles,cache-misses}:S' kill (test-record-group-sampling2) perf record -D kill (test-record-no-delay) perf record -i kill (test-record-no-inherit) perf record -n kill (test-record-no-samples) diff --git a/tools/perf/tests/attr/base-record b/tools/perf/tests/shell/attr/base-record index b44e4e6e4443..b44e4e6e4443 100644 --- a/tools/perf/tests/attr/base-record +++ b/tools/perf/tests/shell/attr/base-record diff --git a/tools/perf/tests/attr/base-record-spe b/tools/perf/tests/shell/attr/base-record-spe index 08fa96b59240..08fa96b59240 100644 --- a/tools/perf/tests/attr/base-record-spe +++ b/tools/perf/tests/shell/attr/base-record-spe diff --git a/tools/perf/tests/attr/base-stat b/tools/perf/tests/shell/attr/base-stat index fccd8ec4d1b0..fccd8ec4d1b0 100644 --- a/tools/perf/tests/attr/base-stat +++ b/tools/perf/tests/shell/attr/base-stat diff --git a/tools/perf/tests/attr/system-wide-dummy b/tools/perf/tests/shell/attr/system-wide-dummy index a1e1d6a263bf..a1e1d6a263bf 100644 --- a/tools/perf/tests/attr/system-wide-dummy +++ b/tools/perf/tests/shell/attr/system-wide-dummy diff --git a/tools/perf/tests/attr/test-record-C0 b/tools/perf/tests/shell/attr/test-record-C0 index 198e8429a1bf..1049ac8b52f2 100644 --- a/tools/perf/tests/attr/test-record-C0 +++ b/tools/perf/tests/shell/attr/test-record-C0 @@ -18,5 +18,7 @@ sample_type=65927 mmap=0 comm=0 task=0 +inherit=0 [event:system-wide-dummy] +inherit=0 diff --git a/tools/perf/tests/attr/test-record-basic b/tools/perf/tests/shell/attr/test-record-basic index b0ca42a5ecc9..b0ca42a5ecc9 100644 --- a/tools/perf/tests/attr/test-record-basic +++ b/tools/perf/tests/shell/attr/test-record-basic diff --git a/tools/perf/tests/attr/test-record-branch-any b/tools/perf/tests/shell/attr/test-record-branch-any index 1a99b3ce6b89..1a99b3ce6b89 100644 --- a/tools/perf/tests/attr/test-record-branch-any +++ b/tools/perf/tests/shell/attr/test-record-branch-any diff --git a/tools/perf/tests/attr/test-record-branch-filter-any b/tools/perf/tests/shell/attr/test-record-branch-filter-any index 709768b508c6..709768b508c6 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-any +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-any diff --git a/tools/perf/tests/attr/test-record-branch-filter-any_call b/tools/perf/tests/shell/attr/test-record-branch-filter-any_call index f943221f7825..f943221f7825 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-any_call +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-any_call diff --git a/tools/perf/tests/attr/test-record-branch-filter-any_ret b/tools/perf/tests/shell/attr/test-record-branch-filter-any_ret index fd4f5b4154a9..fd4f5b4154a9 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-any_ret +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-any_ret diff --git a/tools/perf/tests/attr/test-record-branch-filter-hv b/tools/perf/tests/shell/attr/test-record-branch-filter-hv index 4e52d685ebe1..4e52d685ebe1 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-hv +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-hv diff --git a/tools/perf/tests/attr/test-record-branch-filter-ind_call b/tools/perf/tests/shell/attr/test-record-branch-filter-ind_call index e08c6ab3796e..e08c6ab3796e 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-ind_call +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-ind_call diff --git a/tools/perf/tests/attr/test-record-branch-filter-k b/tools/perf/tests/shell/attr/test-record-branch-filter-k index b4b98f84fc2f..b4b98f84fc2f 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-k +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-k diff --git a/tools/perf/tests/attr/test-record-branch-filter-u b/tools/perf/tests/shell/attr/test-record-branch-filter-u index fb9610edbb0d..fb9610edbb0d 100644 --- a/tools/perf/tests/attr/test-record-branch-filter-u +++ b/tools/perf/tests/shell/attr/test-record-branch-filter-u diff --git a/tools/perf/tests/attr/test-record-count b/tools/perf/tests/shell/attr/test-record-count index 5e9b9019d786..5e9b9019d786 100644 --- a/tools/perf/tests/attr/test-record-count +++ b/tools/perf/tests/shell/attr/test-record-count diff --git a/tools/perf/tests/attr/test-record-data b/tools/perf/tests/shell/attr/test-record-data index a99bb13149c2..a99bb13149c2 100644 --- a/tools/perf/tests/attr/test-record-data +++ b/tools/perf/tests/shell/attr/test-record-data diff --git a/tools/perf/tests/attr/test-record-dummy-C0 b/tools/perf/tests/shell/attr/test-record-dummy-C0 index 576ec48b3aaf..91499405fff4 100644 --- a/tools/perf/tests/attr/test-record-dummy-C0 +++ b/tools/perf/tests/shell/attr/test-record-dummy-C0 @@ -19,7 +19,7 @@ sample_period=4000 sample_type=391 read_format=4|20 disabled=0 -inherit=1 +inherit=0 pinned=0 exclusive=0 exclude_user=0 @@ -37,7 +37,7 @@ precise_ip=0 mmap_data=0 sample_id_all=1 exclude_host=0 -exclude_guest=1 +exclude_guest=0 exclude_callchain_kernel=0 exclude_callchain_user=0 mmap2=1 diff --git a/tools/perf/tests/attr/test-record-freq b/tools/perf/tests/shell/attr/test-record-freq index 89e29f6b2ae0..89e29f6b2ae0 100644 --- a/tools/perf/tests/attr/test-record-freq +++ b/tools/perf/tests/shell/attr/test-record-freq diff --git a/tools/perf/tests/attr/test-record-graph-default b/tools/perf/tests/shell/attr/test-record-graph-default index f0a18b4ea4f5..f0a18b4ea4f5 100644 --- a/tools/perf/tests/attr/test-record-graph-default +++ b/tools/perf/tests/shell/attr/test-record-graph-default diff --git a/tools/perf/tests/attr/test-record-graph-default-aarch64 b/tools/perf/tests/shell/attr/test-record-graph-default-aarch64 index e98d62efb6f7..e98d62efb6f7 100644 --- a/tools/perf/tests/attr/test-record-graph-default-aarch64 +++ b/tools/perf/tests/shell/attr/test-record-graph-default-aarch64 diff --git a/tools/perf/tests/attr/test-record-graph-dwarf b/tools/perf/tests/shell/attr/test-record-graph-dwarf index ae92061d611d..ae92061d611d 100644 --- a/tools/perf/tests/attr/test-record-graph-dwarf +++ b/tools/perf/tests/shell/attr/test-record-graph-dwarf diff --git a/tools/perf/tests/attr/test-record-graph-fp b/tools/perf/tests/shell/attr/test-record-graph-fp index a6e60e839205..a6e60e839205 100644 --- a/tools/perf/tests/attr/test-record-graph-fp +++ b/tools/perf/tests/shell/attr/test-record-graph-fp diff --git a/tools/perf/tests/attr/test-record-graph-fp-aarch64 b/tools/perf/tests/shell/attr/test-record-graph-fp-aarch64 index cbeea9971285..cbeea9971285 100644 --- a/tools/perf/tests/attr/test-record-graph-fp-aarch64 +++ b/tools/perf/tests/shell/attr/test-record-graph-fp-aarch64 diff --git a/tools/perf/tests/attr/test-record-group-sampling b/tools/perf/tests/shell/attr/test-record-group-sampling index 97e7e64a38f0..86a940d7895d 100644 --- a/tools/perf/tests/attr/test-record-group-sampling +++ b/tools/perf/tests/shell/attr/test-record-group-sampling @@ -2,6 +2,7 @@ command = record args = --no-bpf-event -e '{cycles,cache-misses}:S' kill >/dev/null 2>&1 ret = 1 +kernel_until = 6.12 [event-1:base-record] fd=1 @@ -18,7 +19,7 @@ group_fd=1 type=0 config=3 -# default | PERF_SAMPLE_READ +# default | PERF_SAMPLE_READ | PERF_SAMPLE_PERIOD sample_type=343 # PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST diff --git a/tools/perf/tests/shell/attr/test-record-group-sampling1 b/tools/perf/tests/shell/attr/test-record-group-sampling1 new file mode 100644 index 000000000000..4748ab7bf684 --- /dev/null +++ b/tools/perf/tests/shell/attr/test-record-group-sampling1 @@ -0,0 +1,50 @@ +[config] +command = record +args = --no-bpf-event -e '{cycles,cache-misses}:S' kill >/dev/null 2>&1 +ret = 1 +kernel_since = 6.12 + +[event-1:base-record] +fd=1 +group_fd=-1 + +# cycles +type=0 +config=0 + +# default | PERF_SAMPLE_READ | PERF_SAMPLE_PERIOD +sample_type=343 + +# PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING +read_format=28|31 +task=1 +mmap=1 +comm=1 +enable_on_exec=1 +disabled=1 + +# inherit is enabled for group sampling +inherit=1 + +[event-2:base-record] +fd=2 +group_fd=1 + +# cache-misses +type=0 +config=3 + +# default | PERF_SAMPLE_READ | PERF_SAMPLE_PERIOD +sample_type=343 + +# PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING +read_format=28|31 +task=0 +mmap=0 +comm=0 +enable_on_exec=0 +disabled=0 +freq=0 + +# inherit is enabled for group sampling +inherit=1 diff --git a/tools/perf/tests/shell/attr/test-record-group-sampling2 b/tools/perf/tests/shell/attr/test-record-group-sampling2 new file mode 100644 index 000000000000..e0432244a0eb --- /dev/null +++ b/tools/perf/tests/shell/attr/test-record-group-sampling2 @@ -0,0 +1,61 @@ +[config] +command = record +args = --no-bpf-event -c 10000 -e '{cycles,cache-misses}:S' kill >/dev/null 2>&1 +ret = 1 +kernel_since = 6.12 + +[event-1:base-record] +fd=1 +group_fd=-1 + +# cycles +type=0 +config=0 + +# default | PERF_SAMPLE_READ +sample_type=87 + +# PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING +read_format=28|31 +task=1 +mmap=1 +comm=1 +enable_on_exec=1 +disabled=1 + +# inherit is enabled for group sampling +inherit=1 + +# sampling disabled +sample_freq=0 +sample_period=10000 +freq=0 +write_backward=0 + +[event-2:base-record] +fd=2 +group_fd=1 + +# cache-misses +type=0 +config=3 + +# default | PERF_SAMPLE_READ +sample_type=87 + +# PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING +read_format=28|31 +task=0 +mmap=0 +comm=0 +enable_on_exec=0 +disabled=0 + +# inherit is enabled for group sampling +inherit=1 + +# sampling disabled +sample_freq=0 +sample_period=0 +freq=0 +write_backward=0 diff --git a/tools/perf/tests/attr/test-record-group1 b/tools/perf/tests/shell/attr/test-record-group1 index eeb1db392bc9..eeb1db392bc9 100644 --- a/tools/perf/tests/attr/test-record-group1 +++ b/tools/perf/tests/shell/attr/test-record-group1 diff --git a/tools/perf/tests/attr/test-record-group2 b/tools/perf/tests/shell/attr/test-record-group2 index cebdaa8e64e4..891d41a7bddf 100644 --- a/tools/perf/tests/attr/test-record-group2 +++ b/tools/perf/tests/shell/attr/test-record-group2 @@ -2,6 +2,7 @@ command = record args = --no-bpf-event -e '{cycles/period=1234000/,instructions/period=6789000/}:S' kill >/dev/null 2>&1 ret = 1 +kernel_until = 6.12 [event-1:base-record] fd=1 diff --git a/tools/perf/tests/shell/attr/test-record-group3 b/tools/perf/tests/shell/attr/test-record-group3 new file mode 100644 index 000000000000..249be884959e --- /dev/null +++ b/tools/perf/tests/shell/attr/test-record-group3 @@ -0,0 +1,31 @@ +[config] +command = record +args = --no-bpf-event -e '{cycles/period=1234000/,instructions/period=6789000/}:S' kill >/dev/null 2>&1 +ret = 1 +kernel_since = 6.12 + +[event-1:base-record] +fd=1 +group_fd=-1 +config=0|1 +sample_period=1234000 +sample_type=87 +read_format=28|31 +disabled=1 +inherit=1 +freq=0 + +[event-2:base-record] +fd=2 +group_fd=1 +config=0|1 +sample_period=6789000 +sample_type=87 +read_format=28|31 +disabled=0 +inherit=1 +mmap=0 +comm=0 +freq=0 +enable_on_exec=0 +task=0 diff --git a/tools/perf/tests/attr/test-record-no-buffering b/tools/perf/tests/shell/attr/test-record-no-buffering index 583dcbb078ba..583dcbb078ba 100644 --- a/tools/perf/tests/attr/test-record-no-buffering +++ b/tools/perf/tests/shell/attr/test-record-no-buffering diff --git a/tools/perf/tests/attr/test-record-no-inherit b/tools/perf/tests/shell/attr/test-record-no-inherit index 15d1dc162e1c..15d1dc162e1c 100644 --- a/tools/perf/tests/attr/test-record-no-inherit +++ b/tools/perf/tests/shell/attr/test-record-no-inherit diff --git a/tools/perf/tests/attr/test-record-no-samples b/tools/perf/tests/shell/attr/test-record-no-samples index 596fbd6d5a2c..596fbd6d5a2c 100644 --- a/tools/perf/tests/attr/test-record-no-samples +++ b/tools/perf/tests/shell/attr/test-record-no-samples diff --git a/tools/perf/tests/attr/test-record-period b/tools/perf/tests/shell/attr/test-record-period index 119101154c5e..119101154c5e 100644 --- a/tools/perf/tests/attr/test-record-period +++ b/tools/perf/tests/shell/attr/test-record-period diff --git a/tools/perf/tests/attr/test-record-pfm-period b/tools/perf/tests/shell/attr/test-record-pfm-period index 368f5b814094..368f5b814094 100644 --- a/tools/perf/tests/attr/test-record-pfm-period +++ b/tools/perf/tests/shell/attr/test-record-pfm-period diff --git a/tools/perf/tests/attr/test-record-raw b/tools/perf/tests/shell/attr/test-record-raw index 13a5f7860c78..13a5f7860c78 100644 --- a/tools/perf/tests/attr/test-record-raw +++ b/tools/perf/tests/shell/attr/test-record-raw diff --git a/tools/perf/tests/attr/test-record-spe-period b/tools/perf/tests/shell/attr/test-record-spe-period index 75f8c9cd8e3f..75f8c9cd8e3f 100644 --- a/tools/perf/tests/attr/test-record-spe-period +++ b/tools/perf/tests/shell/attr/test-record-spe-period diff --git a/tools/perf/tests/attr/test-record-spe-period-term b/tools/perf/tests/shell/attr/test-record-spe-period-term index 8f60a4fec657..8f60a4fec657 100644 --- a/tools/perf/tests/attr/test-record-spe-period-term +++ b/tools/perf/tests/shell/attr/test-record-spe-period-term diff --git a/tools/perf/tests/attr/test-record-spe-physical-address b/tools/perf/tests/shell/attr/test-record-spe-physical-address index 7ebcf5012ce3..7ebcf5012ce3 100644 --- a/tools/perf/tests/attr/test-record-spe-physical-address +++ b/tools/perf/tests/shell/attr/test-record-spe-physical-address diff --git a/tools/perf/tests/attr/test-record-user-regs-no-sve-aarch64 b/tools/perf/tests/shell/attr/test-record-user-regs-no-sve-aarch64 index bed765450ca9..bed765450ca9 100644 --- a/tools/perf/tests/attr/test-record-user-regs-no-sve-aarch64 +++ b/tools/perf/tests/shell/attr/test-record-user-regs-no-sve-aarch64 diff --git a/tools/perf/tests/attr/test-record-user-regs-old-sve-aarch64 b/tools/perf/tests/shell/attr/test-record-user-regs-old-sve-aarch64 index 15ebfc3418e3..15ebfc3418e3 100644 --- a/tools/perf/tests/attr/test-record-user-regs-old-sve-aarch64 +++ b/tools/perf/tests/shell/attr/test-record-user-regs-old-sve-aarch64 diff --git a/tools/perf/tests/attr/test-record-user-regs-sve-aarch64 b/tools/perf/tests/shell/attr/test-record-user-regs-sve-aarch64 index a65113cd7311..a65113cd7311 100644 --- a/tools/perf/tests/attr/test-record-user-regs-sve-aarch64 +++ b/tools/perf/tests/shell/attr/test-record-user-regs-sve-aarch64 diff --git a/tools/perf/tests/attr/test-stat-C0 b/tools/perf/tests/shell/attr/test-stat-C0 index a2c76d10b2bb..a2c76d10b2bb 100644 --- a/tools/perf/tests/attr/test-stat-C0 +++ b/tools/perf/tests/shell/attr/test-stat-C0 diff --git a/tools/perf/tests/attr/test-stat-basic b/tools/perf/tests/shell/attr/test-stat-basic index 69867d049fda..69867d049fda 100644 --- a/tools/perf/tests/attr/test-stat-basic +++ b/tools/perf/tests/shell/attr/test-stat-basic diff --git a/tools/perf/tests/attr/test-stat-default b/tools/perf/tests/shell/attr/test-stat-default index a1e2da0a9a6d..e47fb4944679 100644 --- a/tools/perf/tests/attr/test-stat-default +++ b/tools/perf/tests/shell/attr/test-stat-default @@ -88,98 +88,142 @@ enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-fe-bound (0x8200) +# PERF_TYPE_RAW / topdown-bad-spec (0x8100) [event13:base-stat] fd=13 group_fd=11 type=4 -config=33280 +config=33024 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-be-bound (0x8300) +# PERF_TYPE_RAW / topdown-fe-bound (0x8200) [event14:base-stat] fd=14 group_fd=11 type=4 -config=33536 +config=33280 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-bad-spec (0x8100) +# PERF_TYPE_RAW / topdown-be-bound (0x8300) [event15:base-stat] fd=15 group_fd=11 type=4 -config=33024 +config=33536 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING +# PERF_TYPE_RAW / topdown-heavy-ops (0x8400) [event16:base-stat] fd=16 +group_fd=11 type=4 -config=4109 +config=33792 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ +# PERF_TYPE_RAW / topdown-br-mispredict (0x8500) [event17:base-stat] fd=17 +group_fd=11 type=4 -config=17039629 +config=34048 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD +# PERF_TYPE_RAW / topdown-fetch-lat (0x8600) [event18:base-stat] fd=18 +group_fd=11 type=4 -config=60 +config=34304 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY +# PERF_TYPE_RAW / topdown-mem-bound (0x8700) [event19:base-stat] fd=19 +group_fd=11 type=4 -config=2097421 +config=34560 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK +# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING [event20:base-stat] fd=20 type=4 -config=316 +config=4109 optional=1 -# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ [event21:base-stat] fd=21 type=4 -config=412 +config=17039629 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD [event22:base-stat] fd=22 type=4 -config=572 +config=60 optional=1 -# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY [event23:base-stat] fd=23 type=4 -config=706 +config=2097421 optional=1 -# PERF_TYPE_RAW / UOPS_ISSUED.ANY +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK [event24:base-stat] fd=24 type=4 +config=316 +optional=1 + +# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +[event25:base-stat] +fd=25 +type=4 +config=412 +optional=1 + +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +[event26:base-stat] +fd=26 +type=4 +config=572 +optional=1 + +# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +[event27:base-stat] +fd=27 +type=4 +config=706 +optional=1 + +# PERF_TYPE_RAW / UOPS_ISSUED.ANY +[event28:base-stat] +fd=28 +type=4 config=270 optional=1 diff --git a/tools/perf/tests/attr/test-stat-detailed-1 b/tools/perf/tests/shell/attr/test-stat-detailed-1 index 1c52cb05c900..3d500d3e0c5c 100644 --- a/tools/perf/tests/attr/test-stat-detailed-1 +++ b/tools/perf/tests/shell/attr/test-stat-detailed-1 @@ -90,99 +90,143 @@ enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-fe-bound (0x8200) +# PERF_TYPE_RAW / topdown-bad-spec (0x8100) [event13:base-stat] fd=13 group_fd=11 type=4 -config=33280 +config=33024 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-be-bound (0x8300) +# PERF_TYPE_RAW / topdown-fe-bound (0x8200) [event14:base-stat] fd=14 group_fd=11 type=4 -config=33536 +config=33280 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-bad-spec (0x8100) +# PERF_TYPE_RAW / topdown-be-bound (0x8300) [event15:base-stat] fd=15 group_fd=11 type=4 -config=33024 +config=33536 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING +# PERF_TYPE_RAW / topdown-heavy-ops (0x8400) [event16:base-stat] fd=16 +group_fd=11 type=4 -config=4109 +config=33792 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ +# PERF_TYPE_RAW / topdown-br-mispredict (0x8500) [event17:base-stat] fd=17 +group_fd=11 type=4 -config=17039629 +config=34048 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD +# PERF_TYPE_RAW / topdown-fetch-lat (0x8600) [event18:base-stat] fd=18 +group_fd=11 type=4 -config=60 +config=34304 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY +# PERF_TYPE_RAW / topdown-mem-bound (0x8700) [event19:base-stat] fd=19 +group_fd=11 type=4 -config=2097421 +config=34560 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK +# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING [event20:base-stat] fd=20 type=4 -config=316 +config=4109 optional=1 -# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ [event21:base-stat] fd=21 type=4 -config=412 +config=17039629 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD [event22:base-stat] fd=22 type=4 -config=572 +config=60 optional=1 -# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY [event23:base-stat] fd=23 type=4 -config=706 +config=2097421 optional=1 -# PERF_TYPE_RAW / UOPS_ISSUED.ANY +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK [event24:base-stat] fd=24 type=4 +config=316 +optional=1 + +# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +[event25:base-stat] +fd=25 +type=4 +config=412 +optional=1 + +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +[event26:base-stat] +fd=26 +type=4 +config=572 +optional=1 + +# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +[event27:base-stat] +fd=27 +type=4 +config=706 +optional=1 + +# PERF_TYPE_RAW / UOPS_ISSUED.ANY +[event28:base-stat] +fd=28 +type=4 config=270 optional=1 @@ -190,8 +234,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event25:base-stat] -fd=25 +[event29:base-stat] +fd=29 type=3 config=0 optional=1 @@ -200,8 +244,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event26:base-stat] -fd=26 +[event30:base-stat] +fd=30 type=3 config=65536 optional=1 @@ -210,8 +254,8 @@ optional=1 # PERF_COUNT_HW_CACHE_LL << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event27:base-stat] -fd=27 +[event31:base-stat] +fd=31 type=3 config=2 optional=1 @@ -220,8 +264,8 @@ optional=1 # PERF_COUNT_HW_CACHE_LL << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event28:base-stat] -fd=28 +[event32:base-stat] +fd=32 type=3 config=65538 optional=1 diff --git a/tools/perf/tests/attr/test-stat-detailed-2 b/tools/perf/tests/shell/attr/test-stat-detailed-2 index 7e961d24a885..01777a63752f 100644 --- a/tools/perf/tests/attr/test-stat-detailed-2 +++ b/tools/perf/tests/shell/attr/test-stat-detailed-2 @@ -90,99 +90,143 @@ enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-fe-bound (0x8200) +# PERF_TYPE_RAW / topdown-bad-spec (0x8100) [event13:base-stat] fd=13 group_fd=11 type=4 -config=33280 +config=33024 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-be-bound (0x8300) +# PERF_TYPE_RAW / topdown-fe-bound (0x8200) [event14:base-stat] fd=14 group_fd=11 type=4 -config=33536 +config=33280 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-bad-spec (0x8100) +# PERF_TYPE_RAW / topdown-be-bound (0x8300) [event15:base-stat] fd=15 group_fd=11 type=4 -config=33024 +config=33536 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING +# PERF_TYPE_RAW / topdown-heavy-ops (0x8400) [event16:base-stat] fd=16 +group_fd=11 type=4 -config=4109 +config=33792 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ +# PERF_TYPE_RAW / topdown-br-mispredict (0x8500) [event17:base-stat] fd=17 +group_fd=11 type=4 -config=17039629 +config=34048 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD +# PERF_TYPE_RAW / topdown-fetch-lat (0x8600) [event18:base-stat] fd=18 +group_fd=11 type=4 -config=60 +config=34304 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY +# PERF_TYPE_RAW / topdown-mem-bound (0x8700) [event19:base-stat] fd=19 +group_fd=11 type=4 -config=2097421 +config=34560 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK +# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING [event20:base-stat] fd=20 type=4 -config=316 +config=4109 optional=1 -# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ [event21:base-stat] fd=21 type=4 -config=412 +config=17039629 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD [event22:base-stat] fd=22 type=4 -config=572 +config=60 optional=1 -# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY [event23:base-stat] fd=23 type=4 -config=706 +config=2097421 optional=1 -# PERF_TYPE_RAW / UOPS_ISSUED.ANY +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK [event24:base-stat] fd=24 type=4 +config=316 +optional=1 + +# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +[event25:base-stat] +fd=25 +type=4 +config=412 +optional=1 + +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +[event26:base-stat] +fd=26 +type=4 +config=572 +optional=1 + +# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +[event27:base-stat] +fd=27 +type=4 +config=706 +optional=1 + +# PERF_TYPE_RAW / UOPS_ISSUED.ANY +[event28:base-stat] +fd=28 +type=4 config=270 optional=1 @@ -190,8 +234,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event25:base-stat] -fd=25 +[event29:base-stat] +fd=29 type=3 config=0 optional=1 @@ -200,8 +244,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event26:base-stat] -fd=26 +[event30:base-stat] +fd=30 type=3 config=65536 optional=1 @@ -210,8 +254,8 @@ optional=1 # PERF_COUNT_HW_CACHE_LL << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event27:base-stat] -fd=27 +[event31:base-stat] +fd=31 type=3 config=2 optional=1 @@ -220,8 +264,8 @@ optional=1 # PERF_COUNT_HW_CACHE_LL << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event28:base-stat] -fd=28 +[event32:base-stat] +fd=32 type=3 config=65538 optional=1 @@ -230,8 +274,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1I << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event29:base-stat] -fd=29 +[event33:base-stat] +fd=33 type=3 config=1 optional=1 @@ -240,8 +284,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1I << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event30:base-stat] -fd=30 +[event34:base-stat] +fd=34 type=3 config=65537 optional=1 @@ -250,8 +294,8 @@ optional=1 # PERF_COUNT_HW_CACHE_DTLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event31:base-stat] -fd=31 +[event35:base-stat] +fd=35 type=3 config=3 optional=1 @@ -260,8 +304,8 @@ optional=1 # PERF_COUNT_HW_CACHE_DTLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event32:base-stat] -fd=32 +[event36:base-stat] +fd=36 type=3 config=65539 optional=1 @@ -270,8 +314,8 @@ optional=1 # PERF_COUNT_HW_CACHE_ITLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event33:base-stat] -fd=33 +[event37:base-stat] +fd=37 type=3 config=4 optional=1 @@ -280,8 +324,8 @@ optional=1 # PERF_COUNT_HW_CACHE_ITLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event34:base-stat] -fd=34 +[event38:base-stat] +fd=38 type=3 config=65540 optional=1 diff --git a/tools/perf/tests/attr/test-stat-detailed-3 b/tools/perf/tests/shell/attr/test-stat-detailed-3 index e50535f45977..8400abd7e1e4 100644 --- a/tools/perf/tests/attr/test-stat-detailed-3 +++ b/tools/perf/tests/shell/attr/test-stat-detailed-3 @@ -90,99 +90,143 @@ enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-fe-bound (0x8200) +# PERF_TYPE_RAW / topdown-bad-spec (0x8100) [event13:base-stat] fd=13 group_fd=11 type=4 -config=33280 +config=33024 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-be-bound (0x8300) +# PERF_TYPE_RAW / topdown-fe-bound (0x8200) [event14:base-stat] fd=14 group_fd=11 type=4 -config=33536 +config=33280 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / topdown-bad-spec (0x8100) +# PERF_TYPE_RAW / topdown-be-bound (0x8300) [event15:base-stat] fd=15 group_fd=11 type=4 -config=33024 +config=33536 disabled=0 enable_on_exec=0 read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING +# PERF_TYPE_RAW / topdown-heavy-ops (0x8400) [event16:base-stat] fd=16 +group_fd=11 type=4 -config=4109 +config=33792 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ +# PERF_TYPE_RAW / topdown-br-mispredict (0x8500) [event17:base-stat] fd=17 +group_fd=11 type=4 -config=17039629 +config=34048 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD +# PERF_TYPE_RAW / topdown-fetch-lat (0x8600) [event18:base-stat] fd=18 +group_fd=11 type=4 -config=60 +config=34304 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY +# PERF_TYPE_RAW / topdown-mem-bound (0x8700) [event19:base-stat] fd=19 +group_fd=11 type=4 -config=2097421 +config=34560 +disabled=0 +enable_on_exec=0 +read_format=15 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK +# PERF_TYPE_RAW / INT_MISC.UOP_DROPPING [event20:base-stat] fd=20 type=4 -config=316 +config=4109 optional=1 -# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +# PERF_TYPE_RAW / cpu/INT_MISC.RECOVERY_CYCLES,cmask=1,edge/ [event21:base-stat] fd=21 type=4 -config=412 +config=17039629 optional=1 -# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.THREAD [event22:base-stat] fd=22 type=4 -config=572 +config=60 optional=1 -# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +# PERF_TYPE_RAW / INT_MISC.RECOVERY_CYCLES_ANY [event23:base-stat] fd=23 type=4 -config=706 +config=2097421 optional=1 -# PERF_TYPE_RAW / UOPS_ISSUED.ANY +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.REF_XCLK [event24:base-stat] fd=24 type=4 +config=316 +optional=1 + +# PERF_TYPE_RAW / IDQ_UOPS_NOT_DELIVERED.CORE +[event25:base-stat] +fd=25 +type=4 +config=412 +optional=1 + +# PERF_TYPE_RAW / CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE +[event26:base-stat] +fd=26 +type=4 +config=572 +optional=1 + +# PERF_TYPE_RAW / UOPS_RETIRED.RETIRE_SLOTS +[event27:base-stat] +fd=27 +type=4 +config=706 +optional=1 + +# PERF_TYPE_RAW / UOPS_ISSUED.ANY +[event28:base-stat] +fd=28 +type=4 config=270 optional=1 @@ -190,8 +234,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event25:base-stat] -fd=25 +[event29:base-stat] +fd=29 type=3 config=0 optional=1 @@ -200,8 +244,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event26:base-stat] -fd=26 +[event30:base-stat] +fd=30 type=3 config=65536 optional=1 @@ -210,8 +254,8 @@ optional=1 # PERF_COUNT_HW_CACHE_LL << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event27:base-stat] -fd=27 +[event31:base-stat] +fd=31 type=3 config=2 optional=1 @@ -220,8 +264,8 @@ optional=1 # PERF_COUNT_HW_CACHE_LL << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event28:base-stat] -fd=28 +[event32:base-stat] +fd=32 type=3 config=65538 optional=1 @@ -230,8 +274,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1I << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event29:base-stat] -fd=29 +[event33:base-stat] +fd=33 type=3 config=1 optional=1 @@ -240,8 +284,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1I << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event30:base-stat] -fd=30 +[event34:base-stat] +fd=34 type=3 config=65537 optional=1 @@ -250,8 +294,8 @@ optional=1 # PERF_COUNT_HW_CACHE_DTLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event31:base-stat] -fd=31 +[event35:base-stat] +fd=35 type=3 config=3 optional=1 @@ -260,8 +304,8 @@ optional=1 # PERF_COUNT_HW_CACHE_DTLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event32:base-stat] -fd=32 +[event36:base-stat] +fd=36 type=3 config=65539 optional=1 @@ -270,8 +314,8 @@ optional=1 # PERF_COUNT_HW_CACHE_ITLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event33:base-stat] -fd=33 +[event37:base-stat] +fd=37 type=3 config=4 optional=1 @@ -280,8 +324,8 @@ optional=1 # PERF_COUNT_HW_CACHE_ITLB << 0 | # (PERF_COUNT_HW_CACHE_OP_READ << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event34:base-stat] -fd=34 +[event38:base-stat] +fd=38 type=3 config=65540 optional=1 @@ -290,8 +334,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | # (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) -[event35:base-stat] -fd=35 +[event39:base-stat] +fd=39 type=3 config=512 optional=1 @@ -300,8 +344,8 @@ optional=1 # PERF_COUNT_HW_CACHE_L1D << 0 | # (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | # (PERF_COUNT_HW_CACHE_RESULT_MISS << 16) -[event36:base-stat] -fd=36 +[event40:base-stat] +fd=40 type=3 config=66048 optional=1 diff --git a/tools/perf/tests/attr/test-stat-group1 b/tools/perf/tests/shell/attr/test-stat-group1 index 1746751123dc..1746751123dc 100644 --- a/tools/perf/tests/attr/test-stat-group1 +++ b/tools/perf/tests/shell/attr/test-stat-group1 diff --git a/tools/perf/tests/attr/test-stat-no-inherit b/tools/perf/tests/shell/attr/test-stat-no-inherit index 924fbb9300d1..924fbb9300d1 100644 --- a/tools/perf/tests/attr/test-stat-no-inherit +++ b/tools/perf/tests/shell/attr/test-stat-no-inherit diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop.sh b/tools/perf/tests/shell/coresight/asm_pure_loop.sh index 2d65defb7e0f..c63bc8c73e26 100755 --- a/tools/perf/tests/shell/coresight/asm_pure_loop.sh +++ b/tools/perf/tests/shell/coresight/asm_pure_loop.sh @@ -1,5 +1,5 @@ #!/bin/sh -e -# CoreSight / ASM Pure Loop +# CoreSight / ASM Pure Loop (exclusive) # SPDX-License-Identifier: GPL-2.0 # Carsten Haitzler <carsten.haitzler@arm.com>, 2021 diff --git a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh index ddcc9bb850f5..8e29630957c8 100755 --- a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh +++ b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh @@ -1,5 +1,5 @@ #!/bin/sh -e -# CoreSight / Memcpy 16k 10 Threads +# CoreSight / Memcpy 16k 10 Threads (exclusive) # SPDX-License-Identifier: GPL-2.0 # Carsten Haitzler <carsten.haitzler@arm.com>, 2021 diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh index 2ce5e139b2fd..0c4c82a1c8e1 100755 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh @@ -1,5 +1,5 @@ #!/bin/sh -e -# CoreSight / Thread Loop 10 Threads - Check TID +# CoreSight / Thread Loop 10 Threads - Check TID (exclusive) # SPDX-License-Identifier: GPL-2.0 # Carsten Haitzler <carsten.haitzler@arm.com>, 2021 diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh index 3ad9498753d7..d3aea9fc6ced 100755 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh @@ -1,5 +1,5 @@ #!/bin/sh -e -# CoreSight / Thread Loop 2 Threads - Check TID +# CoreSight / Thread Loop 2 Threads - Check TID (exclusive) # SPDX-License-Identifier: GPL-2.0 # Carsten Haitzler <carsten.haitzler@arm.com>, 2021 diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh index 4fbb4a29aad3..7429d3a2ae43 100755 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh @@ -1,5 +1,5 @@ #!/bin/sh -e -# CoreSight / Unroll Loop Thread 10 +# CoreSight / Unroll Loop Thread 10 (exclusive) # SPDX-License-Identifier: GPL-2.0 # Carsten Haitzler <carsten.haitzler@arm.com>, 2021 diff --git a/tools/perf/tests/shell/ftrace.sh b/tools/perf/tests/shell/ftrace.sh index a6ee740f0d7e..2df05052c324 100755 --- a/tools/perf/tests/shell/ftrace.sh +++ b/tools/perf/tests/shell/ftrace.sh @@ -67,12 +67,12 @@ test_ftrace_latency() { test_ftrace_profile() { echo "perf ftrace profile test" - perf ftrace profile sleep 0.1 > "${output}" + perf ftrace profile -m 16M sleep 0.1 > "${output}" grep ^# "${output}" grep sleep "${output}" grep schedule "${output}" grep execve "${output}" - time_re="[[:space:]]+10[[:digit:]]{4}\.[[:digit:]]{3}" + time_re="[[:space:]]+1[[:digit:]]{5}\.[[:digit:]]{3}" # 100283.000 100283.000 100283.000 1 __x64_sys_clock_nanosleep # Check for one *clock_nanosleep line with a Count of just 1 that takes a bit more than 0.1 seconds # Strip the _x64_sys part to work with other architectures diff --git a/tools/perf/tests/attr.py b/tools/perf/tests/shell/lib/attr.py index e890c261ad26..3db9a7d78715 100644 --- a/tools/perf/tests/attr.py +++ b/tools/perf/tests/shell/lib/attr.py @@ -246,6 +246,23 @@ class Test(object): return False return True + def restore_sample_rate(self, value=10000): + try: + # Check value of sample_rate + with open("/proc/sys/kernel/perf_event_max_sample_rate", "r") as fIn: + curr_value = fIn.readline() + # If too low restore to reasonable value + if not curr_value or int(curr_value) < int(value): + with open("/proc/sys/kernel/perf_event_max_sample_rate", "w") as fOut: + fOut.write(str(value)) + + except IOError as e: + log.warning("couldn't restore sample_rate value: I/O error %s" % e) + except ValueError as e: + log.warning("couldn't restore sample_rate value: Value error %s" % e) + except TypeError as e: + log.warning("couldn't restore sample_rate value: Type error %s" % e) + def load_events(self, path, events): parser_event = configparser.ConfigParser() parser_event.read(path) @@ -283,6 +300,7 @@ class Test(object): if self.skip_test_kernel_until(): raise Notest(self, "new kernel skip") + self.restore_sample_rate() cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir, self.perf, self.command, tempdir, self.args) ret = os.WEXITSTATUS(os.system(cmd)) diff --git a/tools/perf/tests/shell/lib/coresight.sh b/tools/perf/tests/shell/lib/coresight.sh index 11ed2c25ed91..184d62e7e5bd 100644 --- a/tools/perf/tests/shell/lib/coresight.sh +++ b/tools/perf/tests/shell/lib/coresight.sh @@ -18,7 +18,7 @@ BIN="$DIR/$TEST" # If the test tool/binary does not exist and is executable then skip the test if ! test -x "$BIN"; then exit 2; fi # If CoreSight is not available, skip the test -perf list cs_etm | grep -q cs_etm || exit 2 +perf list pmu | grep -q cs_etm || exit 2 DATD="." # If the data dir env is set then make the data dir use that instead of ./ if test -n "$PERF_TEST_CORESIGHT_DATADIR"; then diff --git a/tools/perf/tests/shell/lib/perf_json_output_lint.py b/tools/perf/tests/shell/lib/perf_json_output_lint.py index abc1fd737782..8ddb85586131 100644 --- a/tools/perf/tests/shell/lib/perf_json_output_lint.py +++ b/tools/perf/tests/shell/lib/perf_json_output_lint.py @@ -57,6 +57,7 @@ def check_json_output(expected_items): 'interval': lambda x: isfloat(x), 'metric-unit': lambda x: True, 'metric-value': lambda x: isfloat(x), + 'metric-threshold': lambda x: x in ['unknown', 'good', 'less good', 'nearly bad', 'bad'], 'metricgroup': lambda x: True, 'node': lambda x: True, 'pcnt-running': lambda x: isfloat(x), @@ -68,13 +69,15 @@ def check_json_output(expected_items): for item in json.loads(input): if expected_items != -1: count = len(item) - if count != expected_items and count >= 1 and count <= 6 and 'metric-value' in item: + if count != expected_items and count >= 1 and count <= 7 and 'metric-value' in item: # Events that generate >1 metric may have isolated metric # values and possibly other prefixes like interval, core, # aggregate-number, or event-runtime/pcnt-running from multiplexing. pass elif count != expected_items and count >= 1 and count <= 5 and 'metricgroup' in item: pass + elif count == expected_items + 1 and 'metric-threshold' in item: + pass elif count != expected_items: raise RuntimeError(f'wrong number of fields. counted {count} expected {expected_items}' f' in \'{item}\'') diff --git a/tools/perf/tests/shell/list.sh b/tools/perf/tests/shell/list.sh index 8a868ae64560..76a9846cff22 100755 --- a/tools/perf/tests/shell/list.sh +++ b/tools/perf/tests/shell/list.sh @@ -24,8 +24,11 @@ trap trap_cleanup EXIT TERM INT test_list_json() { echo "Json output test" + # Generate perf list json output into list_output file. perf list -j -o "${list_output}" - $PYTHON -m json.tool "${list_output}" + # Validate the json using python, redirect the json copy to /dev/null as + # otherwise the test may block writing to stdout. + $PYTHON -m json.tool "${list_output}" /dev/null echo "Json output test [Success]" } diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh index c1ec5762215b..30d195d4c62f 100755 --- a/tools/perf/tests/shell/lock_contention.sh +++ b/tools/perf/tests/shell/lock_contention.sh @@ -27,7 +27,7 @@ check() { exit fi - if ! perf list | grep -q lock:contention_begin; then + if ! perf list tracepoint | grep -q lock:contention_begin; then echo "[Skip] No lock contention tracepoints" err=2 exit diff --git a/tools/perf/tests/shell/perftool-testsuite_report.sh b/tools/perf/tests/shell/perftool-testsuite_report.sh index 973012ce92a7..a8cf75b4e77e 100755 --- a/tools/perf/tests/shell/perftool-testsuite_report.sh +++ b/tools/perf/tests/shell/perftool-testsuite_report.sh @@ -1,5 +1,5 @@ #!/bin/bash -# perftool-testsuite_report +# perftool-testsuite_report (exclusive) # SPDX-License-Identifier: GPL-2.0 test -d "$(dirname "$0")/base_report" || exit 2 diff --git a/tools/perf/tests/shell/pipe_test.sh b/tools/perf/tests/shell/pipe_test.sh index d4c8005ce9b9..e459aa99a951 100755 --- a/tools/perf/tests/shell/pipe_test.sh +++ b/tools/perf/tests/shell/pipe_test.sh @@ -13,6 +13,7 @@ skip_test_missing_symbol ${sym} data=$(mktemp /tmp/perf.data.XXXXXX) data2=$(mktemp /tmp/perf.data2.XXXXXX) prog="perf test -w noploop" +[ "$(uname -m)" = "s390x" ] && prog="$prog 3" err=0 set -e diff --git a/tools/perf/tests/shell/probe_vfs_getname.sh b/tools/perf/tests/shell/probe_vfs_getname.sh index 554e12e83c55..0c5aacc446b3 100755 --- a/tools/perf/tests/shell/probe_vfs_getname.sh +++ b/tools/perf/tests/shell/probe_vfs_getname.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Add vfs_getname probe to get syscall args filenames +# Add vfs_getname probe to get syscall args filenames (exclusive) # SPDX-License-Identifier: GPL-2.0 # Arnaldo Carvalho de Melo <acme@kernel.org>, 2017 diff --git a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh index f38c8ead0b03..47a26f25db9f 100755 --- a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh +++ b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh @@ -40,8 +40,8 @@ trace_libc_inet_pton_backtrace() { case "$(uname -m)" in s390x) eventattr='call-graph=dwarf,max-stack=4' - echo "(__GI_)?getaddrinfo\+0x[[:xdigit:]]+[[:space:]]\($libc|inlined\)$" >> $expected - echo "main\+0x[[:xdigit:]]+[[:space:]]\(.*/bin/ping.*\)$" >> $expected + echo "((__GI_)?getaddrinfo|text_to_binary_address)\+0x[[:xdigit:]]+[[:space:]]\($libc|inlined\)$" >> $expected + echo "(gaih_inet|main)\+0x[[:xdigit:]]+[[:space:]]\(inlined|.*/bin/ping.*\)$" >> $expected ;; ppc64|ppc64le) eventattr='max-stack=4' diff --git a/tools/perf/tests/shell/record+script_probe_vfs_getname.sh b/tools/perf/tests/shell/record+script_probe_vfs_getname.sh index 9a61928e3c9a..5940fdc1df37 100755 --- a/tools/perf/tests/shell/record+script_probe_vfs_getname.sh +++ b/tools/perf/tests/shell/record+script_probe_vfs_getname.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Use vfs_getname probe to get syscall args filenames +# Use vfs_getname probe to get syscall args filenames (exclusive) # Uses the 'perf test shell' library to add probe:vfs_getname to the system # then use it with 'perf record' using 'touch' to write to a temp file, then diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 048078ee2eca..0fc7a909ae9b 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -1,5 +1,5 @@ #!/bin/bash -# perf record tests +# perf record tests (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e @@ -17,6 +17,7 @@ skip_test_missing_symbol ${testsym} err=0 perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +script_output=$(mktemp /tmp/__perf_test.perf.data.XXXXX.script) testprog="perf test -w thloop" cpu_pmu_dir="/sys/bus/event_source/devices/cpu*" br_cntr_file="/caps/branch_counter_nr" @@ -93,7 +94,7 @@ test_per_thread() { test_register_capture() { echo "Register capture test" - if ! perf list | grep -q 'br_inst_retired.near_call' + if ! perf list pmu | grep -q 'br_inst_retired.near_call' then echo "Register capture test [Skipped missing event]" return @@ -228,6 +229,73 @@ test_cgroup() { echo "Cgroup sampling test [Success]" } +test_leader_sampling() { + echo "Basic leader sampling test" + if ! perf record -o "${perfdata}" -e "{instructions,instructions}:Su" -- \ + perf test -w brstack 2> /dev/null + then + echo "Leader sampling [Failed record]" + err=1 + return + fi + index=0 + perf script -i "${perfdata}" > $script_output + while IFS= read -r line + do + # Check if the two instruction counts are equal in each record + instructions=$(echo $line | awk '{for(i=1;i<=NF;i++) if($i=="instructions:") print $(i-1)}') + if [ $(($index%2)) -ne 0 ] && [ ${instructions}x != ${prev_instructions}x ] + then + echo "Leader sampling [Failed inconsistent instructions count]" + err=1 + return + fi + index=$(($index+1)) + prev_instructions=$instructions + done < $script_output + echo "Basic leader sampling test [Success]" +} + +test_topdown_leader_sampling() { + echo "Topdown leader sampling test" + if ! perf stat -e "{slots,topdown-retiring}" true 2> /dev/null + then + echo "Topdown leader sampling [Skipped event parsing failed]" + return + fi + if ! perf record -o "${perfdata}" -e "{instructions,slots,topdown-retiring}:S" true 2> /dev/null + then + echo "Topdown leader sampling [Failed topdown events not reordered correctly]" + err=1 + return + fi + echo "Topdown leader sampling test [Success]" +} + +test_precise_max() { + echo "precise_max attribute test" + if ! perf stat -e "cycles,instructions" true 2> /dev/null + then + echo "precise_max attribute [Skipped no hardware events]" + return + fi + # Just to make sure it doesn't fail + if ! perf record -o "${perfdata}" -e "cycles:P" true 2> /dev/null + then + echo "precise_max attribute [Failed cycles:P event]" + err=1 + return + fi + # On AMD, cycles and instructions events are treated differently + if ! perf record -o "${perfdata}" -e "instructions:P" true 2> /dev/null + then + echo "precise_max attribute [Failed instructions:P event]" + err=1 + return + fi + echo "precise_max attribute test [Success]" +} + # raise the limit of file descriptors to minimum if [[ $default_fd_limit -lt $min_fd_limit ]]; then ulimit -Sn $min_fd_limit @@ -239,6 +307,9 @@ test_system_wide test_workload test_branch_counter test_cgroup +test_leader_sampling +test_topdown_leader_sampling +test_precise_max # restore the default value ulimit -Sn $default_fd_limit diff --git a/tools/perf/tests/shell/record_lbr.sh b/tools/perf/tests/shell/record_lbr.sh index 32314641217e..8d750ee631f8 100755 --- a/tools/perf/tests/shell/record_lbr.sh +++ b/tools/perf/tests/shell/record_lbr.sh @@ -1,5 +1,5 @@ #!/bin/bash -# perf record LBR tests +# perf record LBR tests (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e diff --git a/tools/perf/tests/shell/record_offcpu.sh b/tools/perf/tests/shell/record_offcpu.sh index 67c925f3a15a..678947fe69ee 100755 --- a/tools/perf/tests/shell/record_offcpu.sh +++ b/tools/perf/tests/shell/record_offcpu.sh @@ -1,5 +1,5 @@ #!/bin/sh -# perf record offcpu profiling tests +# perf record offcpu profiling tests (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 3f1e67795490..7a8adf81e4b3 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -73,9 +73,33 @@ test_topdown_groups() { err=1 return fi - if perf stat -e '{topdown-retiring,slots}' true 2>&1 | grep -E -q "<not supported>" + if perf stat -e 'instructions,topdown-retiring,slots' true 2>&1 | grep -E -q "<not supported>" then - echo "Topdown event group test [Failed slots not reordered first]" + echo "Topdown event group test [Failed slots not reordered first in no-group case]" + err=1 + return + fi + if perf stat -e '{instructions,topdown-retiring,slots}' true 2>&1 | grep -E -q "<not supported>" + then + echo "Topdown event group test [Failed slots not reordered first in single group case]" + err=1 + return + fi + if perf stat -e '{instructions,slots},topdown-retiring' true 2>&1 | grep -E -q "<not supported>" + then + echo "Topdown event group test [Failed topdown metrics event not move into slots group]" + err=1 + return + fi + if perf stat -e '{instructions,slots},{topdown-retiring}' true 2>&1 | grep -E -q "<not supported>" + then + echo "Topdown event group test [Failed topdown metrics group not merge into slots group]" + err=1 + return + fi + if perf stat -e '{instructions,r400,r8000}' true 2>&1 | grep -E -q "<not supported>" + then + echo "Topdown event group test [Failed raw format slots not reordered first]" err=1 return fi @@ -117,16 +141,18 @@ test_cputype() { # Find a known PMU for cputype. pmu="" - for i in cpu cpu_atom armv8_pmuv3_0 + devs="/sys/bus/event_source/devices" + for i in $devs/cpu $devs/cpu_atom $devs/armv8_pmuv3_0 $devs/armv8_cortex_* do - if test -d "/sys/devices/$i" + i_base=$(basename "$i") + if test -d "$i" then - pmu="$i" + pmu="$i_base" break fi - if perf stat -e "$i/instructions/" true > /dev/null 2>&1 + if perf stat -e "$i_base/instructions/" true > /dev/null 2>&1 then - pmu="$i" + pmu="$i_base" break fi done @@ -146,6 +172,30 @@ test_cputype() { echo "cputype test [Success]" } +test_hybrid() { + # Test the default stat command on hybrid devices opens one cycles event for + # each CPU type. + echo "hybrid test" + + # Count the number of core PMUs, assume minimum of 1 + pmus=$(ls /sys/bus/event_source/devices/*/cpus 2>/dev/null | wc -l) + if [ "$pmus" -lt 1 ] + then + pmus=1 + fi + + # Run default Perf stat + cycles_events=$(perf stat -- true 2>&1 | grep -E "/cycles/[uH]*| cycles[:uH]* " -c) + + if [ "$pmus" -ne "$cycles_events" ] + then + echo "hybrid test [Found $pmus PMUs but $cycles_events cycles events. Failed]" + err=1 + return + fi + echo "hybrid test [Success]" +} + test_default_stat test_stat_record_report test_stat_record_script @@ -153,4 +203,5 @@ test_stat_repeat_weak_groups test_topdown_groups test_topdown_weak_groups test_cputype +test_hybrid exit $err diff --git a/tools/perf/tests/shell/stat_all_metricgroups.sh b/tools/perf/tests/shell/stat_all_metricgroups.sh index 55ef9c9ded2d..c6d61a4ac3e7 100755 --- a/tools/perf/tests/shell/stat_all_metricgroups.sh +++ b/tools/perf/tests/shell/stat_all_metricgroups.sh @@ -1,9 +1,7 @@ -#!/bin/sh +#!/bin/bash # perf all metricgroups test # SPDX-License-Identifier: GPL-2.0 -set -e - ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] @@ -14,11 +12,37 @@ if ParanoidAndNotRoot 0 then system_wide_flag="" fi - +err=0 for m in $(perf list --raw-dump metricgroups) do echo "Testing $m" - perf stat -M "$m" $system_wide_flag sleep 0.01 + result=$(perf stat -M "$m" $system_wide_flag sleep 0.01 2>&1) + result_err=$? + if [[ $result_err -gt 0 ]] + then + if [[ "$result" =~ \ + "Access to performance monitoring and observability operations is limited" ]] + then + echo "Permission failure" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + elif [[ "$result" =~ "in per-thread mode, enable system wide" ]] + then + echo "Permissions - need system wide mode" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + else + echo "Metric group $m failed" + echo $result + err=1 # Fail + fi + fi done -exit 0 +exit $err diff --git a/tools/perf/tests/shell/stat_all_metrics.sh b/tools/perf/tests/shell/stat_all_metrics.sh index 54774525e18a..73e9347e88a9 100755 --- a/tools/perf/tests/shell/stat_all_metrics.sh +++ b/tools/perf/tests/shell/stat_all_metrics.sh @@ -2,42 +2,87 @@ # perf all metrics test # SPDX-License-Identifier: GPL-2.0 +ParanoidAndNotRoot() +{ + [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] +} + +system_wide_flag="-a" +if ParanoidAndNotRoot 0 +then + system_wide_flag="" +fi + err=0 for m in $(perf list --raw-dump metrics); do echo "Testing $m" - result=$(perf stat -M "$m" true 2>&1) - if [[ "$result" =~ ${m:0:50} ]] || [[ "$result" =~ "<not supported>" ]] + result=$(perf stat -M "$m" $system_wide_flag -- sleep 0.01 2>&1) + result_err=$? + if [[ $result_err -gt 0 ]] then - continue + if [[ "$result" =~ \ + "Access to performance monitoring and observability operations is limited" ]] + then + echo "Permission failure" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + continue + elif [[ "$result" =~ "in per-thread mode, enable system wide" ]] + then + echo "Permissions - need system wide mode" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + continue + elif [[ "$result" =~ "<not supported>" ]] + then + echo "Not supported events" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + continue + elif [[ "$result" =~ "FP_ARITH" || "$result" =~ "AMX" ]] + then + echo "FP issues" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + continue + elif [[ "$result" =~ "PMM" ]] + then + echo "Optane memory issues" + echo $result + if [[ $err -eq 0 ]] + then + err=2 # Skip + fi + continue + fi fi - # Failed so try system wide. - result=$(perf stat -M "$m" -a sleep 0.01 2>&1) + if [[ "$result" =~ ${m:0:50} ]] then continue fi - # Failed again, possibly the workload was too small so retry with something - # longer. - result=$(perf stat -M "$m" perf bench internals synthesize 2>&1) + + # Failed, possibly the workload was too small so retry with something longer. + result=$(perf stat -M "$m" $system_wide_flag -- perf bench internals synthesize 2>&1) if [[ "$result" =~ ${m:0:50} ]] then continue fi echo "Metric '$m' not printed in:" echo "$result" - if [[ "$err" != "1" ]] - then - err=2 - if [[ "$result" =~ "FP_ARITH" || "$result" =~ "AMX" ]] - then - echo "Skip, not fail, for FP issues" - elif [[ "$result" =~ "PMM" ]] - then - echo "Skip, not fail, for Optane memory issues" - else - err=1 - fi - fi + err=1 done exit "$err" diff --git a/tools/perf/tests/shell/stat_all_pmu.sh b/tools/perf/tests/shell/stat_all_pmu.sh index d2a3506e0d19..8b148b300be1 100755 --- a/tools/perf/tests/shell/stat_all_pmu.sh +++ b/tools/perf/tests/shell/stat_all_pmu.sh @@ -1,23 +1,51 @@ -#!/bin/sh -# perf all PMU test +#!/bin/bash +# perf all PMU test (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e +err=0 +result="" + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + echo "$result" + exit 1 +} +trap trap_cleanup EXIT TERM INT # Test all PMU events; however exclude parameterized ones (name contains '?') -for p in $(perf list --raw-dump pmu | sed 's/[[:graph:]]\+?[[:graph:]]\+[[:space:]]//g'); do +for p in $(perf list --raw-dump pmu | sed 's/[[:graph:]]\+?[[:graph:]]\+[[:space:]]//g') +do echo "Testing $p" result=$(perf stat -e "$p" true 2>&1) - if ! echo "$result" | grep -q "$p" && ! echo "$result" | grep -q "<not supported>" ; then - # We failed to see the event and it is supported. Possibly the workload was - # too small so retry with something longer. - result=$(perf stat -e "$p" perf bench internals synthesize 2>&1) - if ! echo "$result" | grep -q "$p" ; then - echo "Event '$p' not printed in:" - echo "$result" - exit 1 - fi + if echo "$result" | grep -q "$p" + then + # Event seen in output. + continue + fi + if echo "$result" | grep -q "<not supported>" + then + # Event not supported, so ignore. + continue + fi + if echo "$result" | grep -q "Access to performance monitoring and observability operations is limited." + then + # Access is limited, so ignore. + continue + fi + + # We failed to see the event and it is supported. Possibly the workload was + # too small so retry with something longer. + result=$(perf stat -e "$p" perf bench internals synthesize 2>&1) + if echo "$result" | grep -q "$p" + then + # Event seen in output. + continue fi + echo "Error: event '$p' not printed in:" + echo "$result" + err=1 done -exit 0 +trap - EXIT TERM INT +exit $err diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh index f250b7d6f773..95d2ad5d17c6 100755 --- a/tools/perf/tests/shell/stat_bpf_counters.sh +++ b/tools/perf/tests/shell/stat_bpf_counters.sh @@ -1,10 +1,10 @@ #!/bin/sh -# perf stat --bpf-counters test +# perf stat --bpf-counters test (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e -workload="perf test -w brstack" +workload="perf test -w sqrtloop" # check whether $2 is within +/- 20% of $1 compare_number() diff --git a/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh b/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh index e75d0780dc78..2ec69060c42f 100755 --- a/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh +++ b/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh @@ -58,22 +58,9 @@ check_system_wide_counted() fi } -check_cpu_list_counted() -{ - check_cpu_list_counted_output=$(perf stat -C 0,1 --bpf-counters --for-each-cgroup ${test_cgroups} -e cpu-clock -x, taskset -c 1 sleep 1 2>&1) - if echo ${check_cpu_list_counted_output} | grep -q -F "<not "; then - echo "Some CPU events are not counted" - if [ "${verbose}" = "1" ]; then - echo ${check_cpu_list_counted_output} - fi - exit 1 - fi -} - check_bpf_counter find_cgroups check_system_wide_counted -check_cpu_list_counted exit 0 diff --git a/tools/perf/tests/shell/test_arm_coresight.sh b/tools/perf/tests/shell/test_arm_coresight.sh index 3302ea0b9672..573af9235b72 100755 --- a/tools/perf/tests/shell/test_arm_coresight.sh +++ b/tools/perf/tests/shell/test_arm_coresight.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Check Arm CoreSight trace data recording and synthesized samples +# Check Arm CoreSight trace data recording and synthesized samples (exclusive) # Uses the 'perf record' to record trace data with Arm CoreSight sinks; # then verify if there have any branch samples and instruction samples @@ -12,7 +12,7 @@ glb_err=0 skip_if_no_cs_etm_event() { - perf list | grep -q 'cs_etm//' && return 0 + perf list pmu | grep -q 'cs_etm//' && return 0 # cs_etm event doesn't exist return 2 diff --git a/tools/perf/tests/shell/test_arm_coresight_disasm.sh b/tools/perf/tests/shell/test_arm_coresight_disasm.sh new file mode 100755 index 000000000000..be2d26303f94 --- /dev/null +++ b/tools/perf/tests/shell/test_arm_coresight_disasm.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Check Arm CoreSight disassembly script completes without errors (exclusive) +# SPDX-License-Identifier: GPL-2.0 + +# The disassembly script reconstructs ranges of instructions and gives these to objdump to +# decode. objdump doesn't like ranges that go backwards, but these are a good indication +# that decoding has gone wrong either in OpenCSD, Perf or in the range reconstruction in +# the script. Test all 3 parts are working correctly by running the script. + +skip_if_no_cs_etm_event() { + perf list pmu | grep -q 'cs_etm//' && return 0 + + # cs_etm event doesn't exist + return 2 +} + +skip_if_no_cs_etm_event || exit 2 + +# Assume an error unless we reach the very end +set -e +glb_err=1 + +perfdata_dir=$(mktemp -d /tmp/__perf_test.perf.data.XXXXX) +perfdata=${perfdata_dir}/perf.data +file=$(mktemp /tmp/temporary_file.XXXXX) +# Relative path works whether it's installed or running from repo +script_path=$(dirname "$0")/../../scripts/python/arm-cs-trace-disasm.py + +cleanup_files() +{ + set +e + rm -rf ${perfdata_dir} + rm -f ${file} + trap - EXIT TERM INT + exit $glb_err +} + +trap cleanup_files EXIT TERM INT + +# Ranges start and end on branches, so check for some likely branch instructions +sep="\s\|\s" +branch_search="\sbl${sep}b${sep}b.ne${sep}b.eq${sep}cbz\s" + +## Test kernel ## +if [ -e /proc/kcore ]; then + echo "Testing kernel disassembly" + perf record -o ${perfdata} -e cs_etm//k --kcore -- touch $file > /dev/null 2>&1 + perf script -i ${perfdata} -s python:${script_path} -- \ + -d --stop-sample=30 2> /dev/null > ${file} + grep -q -e ${branch_search} ${file} + echo "Found kernel branches" +else + # kcore is required for correct kernel decode due to runtime code patching + echo "No kcore, skipping kernel test" +fi + +## Test user ## +echo "Testing userspace disassembly" +perf record -o ${perfdata} -e cs_etm//u -- touch $file > /dev/null 2>&1 +perf script -i ${perfdata} -s python:${script_path} -- \ + -d --stop-sample=30 2> /dev/null > ${file} +grep -q -e ${branch_search} ${file} +echo "Found userspace branches" + +glb_err=0 diff --git a/tools/perf/tests/shell/test_arm_spe.sh b/tools/perf/tests/shell/test_arm_spe.sh index 03d5c7d12ee5..3258368634f7 100755 --- a/tools/perf/tests/shell/test_arm_spe.sh +++ b/tools/perf/tests/shell/test_arm_spe.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Check Arm SPE trace data recording and synthesized samples +# Check Arm SPE trace data recording and synthesized samples (exclusive) # Uses the 'perf record' to record trace data of Arm SPE events; # then verify if any SPE event samples are generated by SPE with @@ -9,7 +9,7 @@ # German Gomez <german.gomez@arm.com>, 2021 skip_if_no_arm_spe_event() { - perf list | grep -E -q 'arm_spe_[0-9]+//' && return 0 + perf list pmu | grep -E -q 'arm_spe_[0-9]+//' && return 0 # arm_spe event doesn't exist return 2 diff --git a/tools/perf/tests/shell/test_arm_spe_fork.sh b/tools/perf/tests/shell/test_arm_spe_fork.sh index 1a7e6a82d0e3..8efeef9fb956 100755 --- a/tools/perf/tests/shell/test_arm_spe_fork.sh +++ b/tools/perf/tests/shell/test_arm_spe_fork.sh @@ -5,7 +5,7 @@ # German Gomez <german.gomez@arm.com>, 2022 skip_if_no_arm_spe_event() { - perf list | grep -E -q 'arm_spe_[0-9]+//' && return 0 + perf list pmu | grep -E -q 'arm_spe_[0-9]+//' && return 0 return 2 } diff --git a/tools/perf/tests/shell/test_data_symbol.sh b/tools/perf/tests/shell/test_data_symbol.sh index 3dfa91832aa8..c86da0235059 100755 --- a/tools/perf/tests/shell/test_data_symbol.sh +++ b/tools/perf/tests/shell/test_data_symbol.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Test data symbol +# Test data symbol (exclusive) # SPDX-License-Identifier: GPL-2.0 # Leo Yan <leo.yan@linaro.org>, 2022 diff --git a/tools/perf/tests/shell/test_intel_pt.sh b/tools/perf/tests/shell/test_intel_pt.sh index 723ec501f99a..e6f0070975f6 100755 --- a/tools/perf/tests/shell/test_intel_pt.sh +++ b/tools/perf/tests/shell/test_intel_pt.sh @@ -1,11 +1,11 @@ #!/bin/sh -# Miscellaneous Intel PT testing +# Miscellaneous Intel PT testing (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e # Skip if no Intel PT -perf list | grep -q 'intel_pt//' || exit 2 +perf list pmu | grep -q 'intel_pt//' || exit 2 shelldir=$(dirname "$0") # shellcheck source=lib/waiting.sh diff --git a/tools/perf/tests/shell/test_stat_intel_tpebs.sh b/tools/perf/tests/shell/test_stat_intel_tpebs.sh index c60b29add980..f95fc64bf0a7 100755 --- a/tools/perf/tests/shell/test_stat_intel_tpebs.sh +++ b/tools/perf/tests/shell/test_stat_intel_tpebs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# test Intel TPEBS counting mode +# test Intel TPEBS counting mode (exclusive) # SPDX-License-Identifier: GPL-2.0 set -e @@ -8,12 +8,15 @@ grep -q GenuineIntel /proc/cpuinfo || { echo Skipping non-Intel; exit 2; } # Use this event for testing because it should exist in all platforms event=cache-misses:R +# Hybrid platforms output like "cpu_atom/cache-misses/R", rather than as above +alt_name=/cache-misses/R + # Without this cmd option, default value or zero is returned -echo "Testing without --record-tpebs" -result=$(perf stat -e "$event" true 2>&1) -[[ "$result" =~ $event ]] || exit 1 +#echo "Testing without --record-tpebs" +#result=$(perf stat -e "$event" true 2>&1) +#[[ "$result" =~ $event || "$result" =~ $alt_name ]] || exit 1 # In platforms that do not support TPEBS, it should execute without error. echo "Testing with --record-tpebs" result=$(perf stat -e "$event" --record-tpebs -a sleep 0.01 2>&1) -[[ "$result" =~ "perf record" && "$result" =~ $event ]] || exit 1 +[[ "$result" =~ "perf record" && "$result" =~ $event || "$result" =~ $alt_name ]] || exit 1 diff --git a/tools/perf/tests/shell/trace+probe_vfs_getname.sh b/tools/perf/tests/shell/trace+probe_vfs_getname.sh index 3146a1eece07..708a13f00635 100755 --- a/tools/perf/tests/shell/trace+probe_vfs_getname.sh +++ b/tools/perf/tests/shell/trace+probe_vfs_getname.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Check open filename arg using perf trace + vfs_getname +# Check open filename arg using perf trace + vfs_getname (exclusive) # Uses the 'perf test shell' library to add probe:vfs_getname to the system # then use it with 'perf trace' using 'touch' to write to a temp file, then @@ -19,7 +19,7 @@ skip_if_no_perf_trace || exit 2 . "$(dirname $0)"/lib/probe_vfs_getname.sh trace_open_vfs_getname() { - evts="$(echo "$(perf list syscalls:sys_enter_open* 2>/dev/null | grep -E 'open(at)? ' | sed -r 's/.*sys_enter_([a-z]+) +\[.*$/\1/')" | sed ':a;N;s:\n:,:g')" + evts="$(echo "$(perf list tracepoint 2>/dev/null | grep -E 'syscalls:sys_enter_open(at)? ' | sed -r 's/.*sys_enter_([a-z]+) +\[.*$/\1/')" | sed ':a;N;s:\n:,:g')" perf trace -e $evts touch $file 2>&1 | \ grep -E " +[0-9]+\.[0-9]+ +\( +[0-9]+\.[0-9]+ ms\): +touch/[0-9]+ open(at)?\((dfd: +CWD, +)?filename: +\"?${file}\"?, +flags: CREAT\|NOCTTY\|NONBLOCK\|WRONLY, +mode: +IRUGO\|IWUGO\) += +[0-9]+$" } diff --git a/tools/perf/tests/shell/trace_exit_race.sh b/tools/perf/tests/shell/trace_exit_race.sh new file mode 100755 index 000000000000..fbb0adc33a88 --- /dev/null +++ b/tools/perf/tests/shell/trace_exit_race.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# perf trace exit race +# SPDX-License-Identifier: GPL-2.0 + +# Check that the last events of a perf trace'd subprocess are not +# lost. Specifically, trace the exiting syscall of "true" 10 times and ensure +# the output contains 10 correct lines. + +# shellcheck source=lib/probe.sh +. "$(dirname $0)"/lib/probe.sh + +skip_if_no_perf_trace || exit 2 + +if [ "$1" = "-v" ]; then + verbose="1" +fi + +iter=10 +regexp=" +[0-9]+\.[0-9]+ [0-9]+ syscalls:sys_enter_exit_group\(\)$" + +trace_shutdown_race() { + for _ in $(seq $iter); do + perf trace --no-comm -e syscalls:sys_enter_exit_group true 2>>$file + done + result="$(grep -c -E "$regexp" $file)" + [ $result = $iter ] +} + + +file=$(mktemp /tmp/temporary_file.XXXXX) + +# Do not use whatever ~/.perfconfig file, it may change the output +# via trace.{show_timestamp,show_prefix,etc} +export PERF_CONFIG=/dev/null + +trace_shutdown_race +err=$? + +if [ $err != 0 ] && [ "${verbose}" = "1" ]; then + lines_not_matching=$(mktemp /tmp/temporary_file.XXXXX) + if grep -v -E "$regexp" $file > $lines_not_matching ; then + echo "Lines not matching the expected regexp: '$regexp':" + cat $lines_not_matching + else + echo "Missing output, expected $iter but only got $result" + fi + rm -f $lines_not_matching +fi + +rm -f ${file} +exit $err diff --git a/tools/perf/tests/task-exit.c b/tools/perf/tests/task-exit.c index d33d0952025c..8e328bbd509d 100644 --- a/tools/perf/tests/task-exit.c +++ b/tools/perf/tests/task-exit.c @@ -152,4 +152,11 @@ out_delete_evlist: return err; } -DEFINE_SUITE("Number of exit events of a simple workload", task_exit); +struct test_case tests__task_exit[] = { + TEST_CASE_EXCLUSIVE("Number of exit events of a simple workload", task_exit), + { .name = NULL, } +}; +struct test_suite suite__task_exit = { + .desc = "Number of exit events of a simple workload", + .test_cases = tests__task_exit, +}; diff --git a/tools/perf/tests/tests-scripts.c b/tools/perf/tests/tests-scripts.c index ed114b044293..cf3ae0c1d871 100644 --- a/tools/perf/tests/tests-scripts.c +++ b/tools/perf/tests/tests-scripts.c @@ -175,6 +175,7 @@ static void append_script(int dir_fd, const char *name, char *desc, struct test_suite *test_suite, **result_tmp; struct test_case *tests; size_t len; + char *exclusive; snprintf(link, sizeof(link), "/proc/%d/fd/%d", getpid(), dir_fd); len = readlink(link, filename, sizeof(filename)); @@ -191,9 +192,13 @@ static void append_script(int dir_fd, const char *name, char *desc, return; } tests[0].name = strdup_check(name); + exclusive = strstr(desc, " (exclusive)"); + if (exclusive != NULL) { + tests[0].exclusive = true; + exclusive[0] = '\0'; + } tests[0].desc = strdup_check(desc); tests[0].run_case = shell_test__run; - test_suite = zalloc(sizeof(*test_suite)); if (!test_suite) { pr_err("Out of memory while building script test suite list\n"); diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h index 6ea2be86b7bf..cb58b43aa063 100644 --- a/tools/perf/tests/tests.h +++ b/tools/perf/tests/tests.h @@ -36,6 +36,7 @@ struct test_case { const char *desc; const char *skip_reason; test_fnptr run_case; + bool exclusive; }; struct test_suite { @@ -62,6 +63,14 @@ struct test_suite { .skip_reason = _reason, \ } +#define TEST_CASE_EXCLUSIVE(description, _name) \ + { \ + .name = #_name, \ + .desc = description, \ + .run_case = test__##_name, \ + .exclusive = true, \ + } + #define DEFINE_SUITE(description, _name) \ struct test_case tests__##_name[] = { \ TEST_CASE(description, _name), \ @@ -83,6 +92,8 @@ DECLARE_SUITE(perf_evsel__tp_sched_test); DECLARE_SUITE(syscall_openat_tp_fields); DECLARE_SUITE(pmu); DECLARE_SUITE(pmu_events); +DECLARE_SUITE(hwmon_pmu); +DECLARE_SUITE(tool_pmu); DECLARE_SUITE(attr); DECLARE_SUITE(dso_data); DECLARE_SUITE(dso_data_cache); diff --git a/tools/perf/tests/tool_pmu.c b/tools/perf/tests/tool_pmu.c new file mode 100644 index 000000000000..187942b749b7 --- /dev/null +++ b/tools/perf/tests/tool_pmu.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "tests.h" +#include "tool_pmu.h" + +static int do_test(enum tool_pmu_event ev, bool with_pmu) +{ + struct evlist *evlist = evlist__new(); + struct evsel *evsel; + struct parse_events_error err; + int ret; + char str[128]; + bool found = false; + + if (!evlist) { + pr_err("evlist allocation failed\n"); + return TEST_FAIL; + } + + if (with_pmu) + snprintf(str, sizeof(str), "tool/%s/", tool_pmu__event_to_str(ev)); + else + snprintf(str, sizeof(str), "%s", tool_pmu__event_to_str(ev)); + + parse_events_error__init(&err); + ret = parse_events(evlist, str, &err); + if (ret) { + if (tool_pmu__skip_event(tool_pmu__event_to_str(ev))) { + ret = TEST_OK; + goto out; + } + + pr_debug("FAILED %s:%d failed to parse event '%s', err %d\n", + __FILE__, __LINE__, str, ret); + parse_events_error__print(&err, str); + ret = TEST_FAIL; + goto out; + } + + ret = TEST_OK; + if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", + __FILE__, __LINE__, str, evlist->core.nr_entries); + ret = TEST_FAIL; + goto out; + } + + evlist__for_each_entry(evlist, evsel) { + if (perf_pmu__is_tool(evsel->pmu)) { + if (evsel->core.attr.config != ev) { + pr_debug("FAILED %s:%d Unexpected config for '%s', %lld != %d\n", + __FILE__, __LINE__, str, evsel->core.attr.config, ev); + ret = TEST_FAIL; + goto out; + } + found = true; + } + } + + if (!found && !tool_pmu__skip_event(tool_pmu__event_to_str(ev))) { + pr_debug("FAILED %s:%d Didn't find tool event '%s' in parsed evsels\n", + __FILE__, __LINE__, str); + ret = TEST_FAIL; + } + +out: + parse_events_error__exit(&err); + evlist__delete(evlist); + return ret; +} + +static int test__tool_pmu_without_pmu(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + int i; + + tool_pmu__for_each_event(i) { + int ret = do_test(i, /*with_pmu=*/false); + + if (ret != TEST_OK) + return ret; + } + return TEST_OK; +} + +static int test__tool_pmu_with_pmu(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + int i; + + tool_pmu__for_each_event(i) { + int ret = do_test(i, /*with_pmu=*/true); + + if (ret != TEST_OK) + return ret; + } + return TEST_OK; +} + +static struct test_case tests__tool_pmu[] = { + TEST_CASE("Parsing without PMU name", tool_pmu_without_pmu), + TEST_CASE("Parsing with PMU name", tool_pmu_with_pmu), + { .name = NULL, } +}; + +struct test_suite suite__tool_pmu = { + .desc = "Tool PMU", + .test_cases = tests__tool_pmu, +}; diff --git a/tools/perf/util/Build b/tools/perf/util/Build index dc616292b2dd..c06d2ee9024c 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -83,6 +83,8 @@ perf-util-y += pmu.o perf-util-y += pmus.o perf-util-y += pmu-flex.o perf-util-y += pmu-bison.o +perf-util-y += hwmon_pmu.o +perf-util-y += tool_pmu.o perf-util-y += svghelper.o perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event-info.o perf-util-y += trace-event-scripting.o @@ -199,11 +201,14 @@ ifndef CONFIG_SETNS perf-util-y += setns.o endif -perf-util-$(CONFIG_DWARF) += probe-finder.o -perf-util-$(CONFIG_DWARF) += dwarf-aux.o -perf-util-$(CONFIG_DWARF) += dwarf-regs.o -perf-util-$(CONFIG_DWARF) += debuginfo.o -perf-util-$(CONFIG_DWARF) += annotate-data.o +perf-util-$(CONFIG_LIBDW) += probe-finder.o +perf-util-$(CONFIG_LIBDW) += dwarf-aux.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-csky.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-powerpc.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-x86.o +perf-util-$(CONFIG_LIBDW) += debuginfo.o +perf-util-$(CONFIG_LIBDW) += annotate-data.o perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind-local.o @@ -234,7 +239,7 @@ perf-util-$(CONFIG_LIBLLVM) += llvm-c-helpers.o ifdef CONFIG_JITDUMP perf-util-$(CONFIG_LIBELF) += jitdump.o perf-util-$(CONFIG_LIBELF) += genelf.o -perf-util-$(CONFIG_DWARF) += genelf_debug.o +perf-util-$(CONFIG_LIBDW) += genelf_debug.o endif perf-util-y += perf-hooks.o diff --git a/tools/perf/util/annotate-data.h b/tools/perf/util/annotate-data.h index 8ac0fd94a0ba..98c80b2268dd 100644 --- a/tools/perf/util/annotate-data.h +++ b/tools/perf/util/annotate-data.h @@ -9,7 +9,7 @@ #include "dwarf-regs.h" #include "annotate.h" -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT #include "debuginfo.h" #endif @@ -165,7 +165,7 @@ struct annotated_data_stat { }; extern struct annotated_data_stat ann_data_stat; -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT /* * Type information in a register, valid when @ok is true. * The @caller_saved registers are invalidated after a function call. @@ -244,7 +244,7 @@ bool get_global_var_info(struct data_loc_info *dloc, u64 addr, const char **var_name, int *var_offset); void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind); -#else /* HAVE_DWARF_SUPPORT */ +#else /* HAVE_LIBDW_SUPPORT */ static inline struct annotated_data_type * find_data_type(struct data_loc_info *dloc __maybe_unused) @@ -276,7 +276,7 @@ static inline int hist_entry__annotate_data_tty(struct hist_entry *he __maybe_un return -1; } -#endif /* HAVE_DWARF_SUPPORT */ +#endif /* HAVE_LIBDW_SUPPORT */ #ifdef HAVE_SLANG_SUPPORT int hist_entry__annotate_data_tui(struct hist_entry *he, struct evsel *evsel, diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 37ce43c4eb8f..32e15c9f53f3 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -2116,6 +2116,12 @@ static int annotation__config(const char *var, const char *value, void *data) opt->offset_level = ANNOTATION__MAX_OFFSET_LEVEL; else if (opt->offset_level < ANNOTATION__MIN_OFFSET_LEVEL) opt->offset_level = ANNOTATION__MIN_OFFSET_LEVEL; + } else if (!strcmp(var, "annotate.disassemblers")) { + opt->disassemblers_str = strdup(value); + if (!opt->disassemblers_str) { + pr_err("Not enough memory for annotate.disassemblers\n"); + return -1; + } } else if (!strcmp(var, "annotate.hide_src_code")) { opt->hide_src_code = perf_config_bool("hide_src_code", value); } else if (!strcmp(var, "annotate.jump_arrows")) { @@ -2292,7 +2298,7 @@ static int extract_reg_offset(struct arch *arch, const char *str, if (regname == NULL) return -1; - op_loc->reg1 = get_dwarf_regnum(regname, 0); + op_loc->reg1 = get_dwarf_regnum(regname, arch->e_machine, arch->e_flags); free(regname); /* Get the second register */ @@ -2305,7 +2311,7 @@ static int extract_reg_offset(struct arch *arch, const char *str, if (regname == NULL) return -1; - op_loc->reg2 = get_dwarf_regnum(regname, 0); + op_loc->reg2 = get_dwarf_regnum(regname, arch->e_machine, arch->e_flags); free(regname); } return 0; @@ -2405,7 +2411,7 @@ int annotate_get_insn_location(struct arch *arch, struct disasm_line *dl, return -1; if (*s == arch->objdump.register_char) - op_loc->reg1 = get_dwarf_regnum(s, 0); + op_loc->reg1 = get_dwarf_regnum(s, arch->e_machine, arch->e_flags); else if (*s == arch->objdump.imm_char) { op_loc->offset = strtol(s + 1, &p, 0); if (p && p != s + 1) diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h index 8b9e05a1932f..194a05cbc506 100644 --- a/tools/perf/util/annotate.h +++ b/tools/perf/util/annotate.h @@ -34,6 +34,9 @@ struct annotated_data_type; #define ANNOTATION__BR_CNTR_WIDTH 30 #define ANNOTATION_DUMMY_LEN 256 +// llvm, capstone, objdump +#define MAX_DISASSEMBLERS 3 + struct annotation_options { bool hide_src_code, use_offset, @@ -49,11 +52,14 @@ struct annotation_options { annotate_src, full_addr; u8 offset_level; + u8 nr_disassemblers; int min_pcnt; int max_lines; int context; char *objdump_path; char *disassembler_style; + const char *disassemblers_str; + const char *disassemblers[MAX_DISASSEMBLERS]; const char *prefix; const char *prefix_strip; unsigned int percent_type; diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h index 1443c28545a9..358c611eeddb 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -56,15 +56,15 @@ enum arm_spe_op_type { ARM_SPE_OP_BR_INDIRECT = 1 << 17, }; -enum arm_spe_neoverse_data_source { - ARM_SPE_NV_L1D = 0x0, - ARM_SPE_NV_L2 = 0x8, - ARM_SPE_NV_PEER_CORE = 0x9, - ARM_SPE_NV_LOCAL_CLUSTER = 0xa, - ARM_SPE_NV_SYS_CACHE = 0xb, - ARM_SPE_NV_PEER_CLUSTER = 0xc, - ARM_SPE_NV_REMOTE = 0xd, - ARM_SPE_NV_DRAM = 0xe, +enum arm_spe_common_data_source { + ARM_SPE_COMMON_DS_L1D = 0x0, + ARM_SPE_COMMON_DS_L2 = 0x8, + ARM_SPE_COMMON_DS_PEER_CORE = 0x9, + ARM_SPE_COMMON_DS_LOCAL_CLUSTER = 0xa, + ARM_SPE_COMMON_DS_SYS_CACHE = 0xb, + ARM_SPE_COMMON_DS_PEER_CLUSTER = 0xc, + ARM_SPE_COMMON_DS_REMOTE = 0xd, + ARM_SPE_COMMON_DS_DRAM = 0xe, }; struct arm_spe_record { diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 138ffc71b32d..dbf13f47879c 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -46,7 +46,6 @@ struct arm_spe { struct perf_session *session; struct machine *machine; u32 pmu_type; - u64 midr; struct perf_tsc_conversion tc; @@ -69,7 +68,7 @@ struct arm_spe { u64 llc_access_id; u64 tlb_miss_id; u64 tlb_access_id; - u64 branch_miss_id; + u64 branch_id; u64 remote_access_id; u64 memory_id; u64 instructions_id; @@ -78,6 +77,11 @@ struct arm_spe { unsigned long num_events; u8 use_ctx_pkt_for_pid; + + u64 **metadata; + u64 metadata_ver; + u64 metadata_nr_cpu; + bool is_homogeneous; }; struct arm_spe_queue { @@ -96,6 +100,7 @@ struct arm_spe_queue { u64 timestamp; struct thread *thread; u64 period_instructions; + u32 flags; }; static void arm_spe_dump(struct arm_spe *spe __maybe_unused, @@ -118,7 +123,7 @@ static void arm_spe_dump(struct arm_spe *spe __maybe_unused, else pkt_len = 1; printf("."); - color_fprintf(stdout, color, " %08x: ", pos); + color_fprintf(stdout, color, " %08zx: ", pos); for (i = 0; i < pkt_len; i++) color_fprintf(stdout, color, " %02x", buf[i]); for (; i < 16; i++) @@ -273,6 +278,20 @@ static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid) return 0; } +static u64 *arm_spe__get_metadata_by_cpu(struct arm_spe *spe, u64 cpu) +{ + u64 i; + + if (!spe->metadata) + return NULL; + + for (i = 0; i < spe->metadata_nr_cpu; i++) + if (spe->metadata[i][ARM_SPE_CPU] == cpu) + return spe->metadata[i]; + + return NULL; +} + static struct simd_flags arm_spe__synth_simd_flags(const struct arm_spe_record *record) { struct simd_flags simd_flags = {}; @@ -376,6 +395,7 @@ static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq, sample.stream_id = spe_events_id; sample.addr = record->to_ip; sample.weight = record->latency; + sample.flags = speq->flags; return arm_spe_deliver_synth_event(spe, speq, event, &sample); } @@ -400,24 +420,44 @@ static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq, sample.id = spe_events_id; sample.stream_id = spe_events_id; - sample.addr = record->virt_addr; + sample.addr = record->to_ip; sample.phys_addr = record->phys_addr; sample.data_src = data_src; sample.period = spe->instructions_sample_period; sample.weight = record->latency; + sample.flags = speq->flags; return arm_spe_deliver_synth_event(spe, speq, event, &sample); } -static const struct midr_range neoverse_spe[] = { +static const struct midr_range common_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A725), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), {}, }; -static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *record, - union perf_mem_data_src *data_src) +static void arm_spe__sample_flags(struct arm_spe_queue *speq) +{ + const struct arm_spe_record *record = &speq->decoder->record; + + speq->flags = 0; + if (record->op & ARM_SPE_OP_BRANCH_ERET) { + speq->flags = PERF_IP_FLAG_BRANCH; + + if (record->type & ARM_SPE_BRANCH_MISS) + speq->flags |= PERF_IP_FLAG_BRANCH_MISS; + } +} + +static void arm_spe__synth_data_source_common(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) { /* * Even though four levels of cache hierarchy are possible, no known @@ -439,17 +479,17 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec } switch (record->source) { - case ARM_SPE_NV_L1D: + case ARM_SPE_COMMON_DS_L1D: data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; data_src->mem_snoop = PERF_MEM_SNOOP_NONE; break; - case ARM_SPE_NV_L2: + case ARM_SPE_COMMON_DS_L2: data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; data_src->mem_snoop = PERF_MEM_SNOOP_NONE; break; - case ARM_SPE_NV_PEER_CORE: + case ARM_SPE_COMMON_DS_PEER_CORE: data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; @@ -458,8 +498,8 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec * We don't know if this is L1, L2 but we do know it was a cache-2-cache * transfer, so set SNOOPX_PEER */ - case ARM_SPE_NV_LOCAL_CLUSTER: - case ARM_SPE_NV_PEER_CLUSTER: + case ARM_SPE_COMMON_DS_LOCAL_CLUSTER: + case ARM_SPE_COMMON_DS_PEER_CLUSTER: data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; @@ -467,7 +507,7 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec /* * System cache is assumed to be L3 */ - case ARM_SPE_NV_SYS_CACHE: + case ARM_SPE_COMMON_DS_SYS_CACHE: data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; data_src->mem_snoop = PERF_MEM_SNOOP_HIT; @@ -476,13 +516,13 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec * We don't know what level it hit in, except it came from the other * socket */ - case ARM_SPE_NV_REMOTE: + case ARM_SPE_COMMON_DS_REMOTE: data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1; data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; break; - case ARM_SPE_NV_DRAM: + case ARM_SPE_COMMON_DS_DRAM: data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; data_src->mem_snoop = PERF_MEM_SNOOP_NONE; @@ -492,8 +532,8 @@ static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *rec } } -static void arm_spe__synth_data_source_generic(const struct arm_spe_record *record, - union perf_mem_data_src *data_src) +static void arm_spe__synth_memory_level(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) { if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) { data_src->mem_lvl = PERF_MEM_LVL_L3; @@ -515,10 +555,55 @@ static void arm_spe__synth_data_source_generic(const struct arm_spe_record *reco data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1; } -static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr) +static bool arm_spe__is_common_ds_encoding(struct arm_spe_queue *speq) +{ + struct arm_spe *spe = speq->spe; + bool is_in_cpu_list; + u64 *metadata = NULL; + u64 midr = 0; + + /* Metadata version 1 assumes all CPUs are the same (old behavior) */ + if (spe->metadata_ver == 1) { + const char *cpuid; + + pr_warning_once("Old SPE metadata, re-record to improve decode accuracy\n"); + cpuid = perf_env__cpuid(spe->session->evlist->env); + midr = strtol(cpuid, NULL, 16); + } else { + /* CPU ID is -1 for per-thread mode */ + if (speq->cpu < 0) { + /* + * On the heterogeneous system, due to CPU ID is -1, + * cannot confirm the data source packet is supported. + */ + if (!spe->is_homogeneous) + return false; + + /* In homogeneous system, simply use CPU0's metadata */ + if (spe->metadata) + metadata = spe->metadata[0]; + } else { + metadata = arm_spe__get_metadata_by_cpu(spe, speq->cpu); + } + + if (!metadata) + return false; + + midr = metadata[ARM_SPE_CPU_MIDR]; + } + + is_in_cpu_list = is_midr_in_range_list(midr, common_ds_encoding_cpus); + if (is_in_cpu_list) + return true; + else + return false; +} + +static u64 arm_spe__synth_data_source(struct arm_spe_queue *speq, + const struct arm_spe_record *record) { union perf_mem_data_src data_src = { .mem_op = PERF_MEM_OP_NA }; - bool is_neoverse = is_midr_in_range_list(midr, neoverse_spe); + bool is_common = arm_spe__is_common_ds_encoding(speq); if (record->op & ARM_SPE_OP_LD) data_src.mem_op = PERF_MEM_OP_LOAD; @@ -527,10 +612,10 @@ static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 m else return 0; - if (is_neoverse) - arm_spe__synth_data_source_neoverse(record, &data_src); + if (is_common) + arm_spe__synth_data_source_common(record, &data_src); else - arm_spe__synth_data_source_generic(record, &data_src); + arm_spe__synth_memory_level(record, &data_src); if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) { data_src.mem_dtlb = PERF_MEM_TLB_WK; @@ -551,7 +636,8 @@ static int arm_spe_sample(struct arm_spe_queue *speq) u64 data_src; int err; - data_src = arm_spe__synth_data_source(record, spe->midr); + arm_spe__sample_flags(speq); + data_src = arm_spe__synth_data_source(speq, record); if (spe->sample_flc) { if (record->type & ARM_SPE_L1D_MISS) { @@ -601,8 +687,8 @@ static int arm_spe_sample(struct arm_spe_queue *speq) } } - if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) { - err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id); + if (spe->sample_branch && (record->op & ARM_SPE_OP_BRANCH_ERET)) { + err = arm_spe__synth_branch_sample(speq, spe->branch_id); if (err) return err; } @@ -1016,6 +1102,73 @@ static int arm_spe_flush(struct perf_session *session __maybe_unused, return 0; } +static u64 *arm_spe__alloc_per_cpu_metadata(u64 *buf, int per_cpu_size) +{ + u64 *metadata; + + metadata = zalloc(per_cpu_size); + if (!metadata) + return NULL; + + memcpy(metadata, buf, per_cpu_size); + return metadata; +} + +static void arm_spe__free_metadata(u64 **metadata, int nr_cpu) +{ + int i; + + for (i = 0; i < nr_cpu; i++) + zfree(&metadata[i]); + free(metadata); +} + +static u64 **arm_spe__alloc_metadata(struct perf_record_auxtrace_info *info, + u64 *ver, int *nr_cpu) +{ + u64 *ptr = (u64 *)info->priv; + u64 metadata_size; + u64 **metadata = NULL; + int hdr_sz, per_cpu_sz, i; + + metadata_size = info->header.size - + sizeof(struct perf_record_auxtrace_info); + + /* Metadata version 1 */ + if (metadata_size == ARM_SPE_AUXTRACE_V1_PRIV_SIZE) { + *ver = 1; + *nr_cpu = 0; + /* No per CPU metadata */ + return NULL; + } + + *ver = ptr[ARM_SPE_HEADER_VERSION]; + hdr_sz = ptr[ARM_SPE_HEADER_SIZE]; + *nr_cpu = ptr[ARM_SPE_CPUS_NUM]; + + metadata = calloc(*nr_cpu, sizeof(*metadata)); + if (!metadata) + return NULL; + + /* Locate the start address of per CPU metadata */ + ptr += hdr_sz; + per_cpu_sz = (metadata_size - (hdr_sz * sizeof(u64))) / (*nr_cpu); + + for (i = 0; i < *nr_cpu; i++) { + metadata[i] = arm_spe__alloc_per_cpu_metadata(ptr, per_cpu_sz); + if (!metadata[i]) + goto err_per_cpu_metadata; + + ptr += per_cpu_sz / sizeof(u64); + } + + return metadata; + +err_per_cpu_metadata: + arm_spe__free_metadata(metadata, *nr_cpu); + return NULL; +} + static void arm_spe_free_queue(void *priv) { struct arm_spe_queue *speq = priv; @@ -1050,6 +1203,7 @@ static void arm_spe_free(struct perf_session *session) auxtrace_heap__free(&spe->heap); arm_spe_free_events(session); session->auxtrace = NULL; + arm_spe__free_metadata(spe->metadata, spe->metadata_nr_cpu); free(spe); } @@ -1061,16 +1215,60 @@ static bool arm_spe_evsel_is_auxtrace(struct perf_session *session, return evsel->core.attr.type == spe->pmu_type; } -static const char * const arm_spe_info_fmts[] = { - [ARM_SPE_PMU_TYPE] = " PMU Type %"PRId64"\n", +static const char * const metadata_hdr_v1_fmts[] = { + [ARM_SPE_PMU_TYPE] = " PMU Type :%"PRId64"\n", + [ARM_SPE_PER_CPU_MMAPS] = " Per CPU mmaps :%"PRId64"\n", +}; + +static const char * const metadata_hdr_fmts[] = { + [ARM_SPE_HEADER_VERSION] = " Header version :%"PRId64"\n", + [ARM_SPE_HEADER_SIZE] = " Header size :%"PRId64"\n", + [ARM_SPE_PMU_TYPE_V2] = " PMU type v2 :%"PRId64"\n", + [ARM_SPE_CPUS_NUM] = " CPU number :%"PRId64"\n", +}; + +static const char * const metadata_per_cpu_fmts[] = { + [ARM_SPE_MAGIC] = " Magic :0x%"PRIx64"\n", + [ARM_SPE_CPU] = " CPU # :%"PRId64"\n", + [ARM_SPE_CPU_NR_PARAMS] = " Num of params :%"PRId64"\n", + [ARM_SPE_CPU_MIDR] = " MIDR :0x%"PRIx64"\n", + [ARM_SPE_CPU_PMU_TYPE] = " PMU Type :%"PRId64"\n", + [ARM_SPE_CAP_MIN_IVAL] = " Min Interval :%"PRId64"\n", }; -static void arm_spe_print_info(__u64 *arr) +static void arm_spe_print_info(struct arm_spe *spe, __u64 *arr) { + unsigned int i, cpu, hdr_size, cpu_num, cpu_size; + const char * const *hdr_fmts; + if (!dump_trace) return; - fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]); + if (spe->metadata_ver == 1) { + cpu_num = 0; + hdr_size = ARM_SPE_AUXTRACE_V1_PRIV_MAX; + hdr_fmts = metadata_hdr_v1_fmts; + } else { + cpu_num = arr[ARM_SPE_CPUS_NUM]; + hdr_size = arr[ARM_SPE_HEADER_SIZE]; + hdr_fmts = metadata_hdr_fmts; + } + + for (i = 0; i < hdr_size; i++) + fprintf(stdout, hdr_fmts[i], arr[i]); + + arr += hdr_size; + for (cpu = 0; cpu < cpu_num; cpu++) { + /* + * The parameters from ARM_SPE_MAGIC to ARM_SPE_CPU_NR_PARAMS + * are fixed. The sequential parameter size is decided by the + * field 'ARM_SPE_CPU_NR_PARAMS'. + */ + cpu_size = (ARM_SPE_CPU_NR_PARAMS + 1) + arr[ARM_SPE_CPU_NR_PARAMS]; + for (i = 0; i < cpu_size; i++) + fprintf(stdout, metadata_per_cpu_fmts[i], arr[i]); + arr += cpu_size; + } } static void arm_spe_set_event_name(struct evlist *evlist, u64 id, @@ -1202,12 +1400,12 @@ arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session) if (spe->synth_opts.branches) { spe->sample_branch = true; - /* Branch miss */ + /* Branch */ err = perf_session__deliver_synth_attr_event(session, &attr, id); if (err) return err; - spe->branch_miss_id = id; - arm_spe_set_event_name(evlist, id, "branch-miss"); + spe->branch_id = id; + arm_spe_set_event_name(evlist, id, "branch"); id += 1; } @@ -1258,24 +1456,57 @@ synth_instructions_out: return 0; } +static bool arm_spe__is_homogeneous(u64 **metadata, int nr_cpu) +{ + u64 midr; + int i; + + if (!nr_cpu) + return false; + + for (i = 0; i < nr_cpu; i++) { + if (!metadata[i]) + return false; + + if (i == 0) { + midr = metadata[i][ARM_SPE_CPU_MIDR]; + continue; + } + + if (midr != metadata[i][ARM_SPE_CPU_MIDR]) + return false; + } + + return true; +} + int arm_spe_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX; + size_t min_sz = ARM_SPE_AUXTRACE_V1_PRIV_SIZE; struct perf_record_time_conv *tc = &session->time_conv; - const char *cpuid = perf_env__cpuid(session->evlist->env); - u64 midr = strtol(cpuid, NULL, 16); struct arm_spe *spe; - int err; + u64 **metadata = NULL; + u64 metadata_ver; + int nr_cpu, err; if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) + min_sz) return -EINVAL; + metadata = arm_spe__alloc_metadata(auxtrace_info, &metadata_ver, + &nr_cpu); + if (!metadata && metadata_ver != 1) { + pr_err("Failed to parse Arm SPE metadata.\n"); + return -EINVAL; + } + spe = zalloc(sizeof(struct arm_spe)); - if (!spe) - return -ENOMEM; + if (!spe) { + err = -ENOMEM; + goto err_free_metadata; + } err = auxtrace_queues__init(&spe->queues); if (err) @@ -1284,8 +1515,14 @@ int arm_spe_process_auxtrace_info(union perf_event *event, spe->session = session; spe->machine = &session->machines.host; /* No kvm support */ spe->auxtrace_type = auxtrace_info->type; - spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; - spe->midr = midr; + if (metadata_ver == 1) + spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; + else + spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE_V2]; + spe->metadata = metadata; + spe->metadata_ver = metadata_ver; + spe->metadata_nr_cpu = nr_cpu; + spe->is_homogeneous = arm_spe__is_homogeneous(metadata, nr_cpu); spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); @@ -1318,7 +1555,7 @@ int arm_spe_process_auxtrace_info(union perf_event *event, spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace; session->auxtrace = &spe->auxtrace; - arm_spe_print_info(&auxtrace_info->priv[0]); + arm_spe_print_info(spe, &auxtrace_info->priv[0]); if (dump_trace) return 0; @@ -1346,5 +1583,7 @@ err_free_queues: session->auxtrace = NULL; err_free: free(spe); +err_free_metadata: + arm_spe__free_metadata(metadata, nr_cpu); return err; } diff --git a/tools/perf/util/arm-spe.h b/tools/perf/util/arm-spe.h index 4f4900c18f3e..390679a4af2f 100644 --- a/tools/perf/util/arm-spe.h +++ b/tools/perf/util/arm-spe.h @@ -12,10 +12,46 @@ enum { ARM_SPE_PMU_TYPE, ARM_SPE_PER_CPU_MMAPS, + ARM_SPE_AUXTRACE_V1_PRIV_MAX, +}; + +#define ARM_SPE_AUXTRACE_V1_PRIV_SIZE \ + (ARM_SPE_AUXTRACE_V1_PRIV_MAX * sizeof(u64)) + +enum { + /* + * The old metadata format (defined above) does not include a + * field for version number. Version 1 is reserved and starts + * from version 2. + */ + ARM_SPE_HEADER_VERSION, + /* Number of sizeof(u64) */ + ARM_SPE_HEADER_SIZE, + /* PMU type shared by CPUs */ + ARM_SPE_PMU_TYPE_V2, + /* Number of CPUs */ + ARM_SPE_CPUS_NUM, ARM_SPE_AUXTRACE_PRIV_MAX, }; -#define ARM_SPE_AUXTRACE_PRIV_SIZE (ARM_SPE_AUXTRACE_PRIV_MAX * sizeof(u64)) +enum { + /* Magic number */ + ARM_SPE_MAGIC, + /* CPU logical number in system */ + ARM_SPE_CPU, + /* Number of parameters */ + ARM_SPE_CPU_NR_PARAMS, + /* CPU MIDR */ + ARM_SPE_CPU_MIDR, + /* Associated PMU type */ + ARM_SPE_CPU_PMU_TYPE, + /* Minimal interval */ + ARM_SPE_CAP_MIN_IVAL, + ARM_SPE_CPU_PRIV_MAX, +}; + +#define ARM_SPE_HEADER_CURRENT_VERSION 2 + union perf_event; struct perf_session; diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h index a1895a4f530b..dddaf4f3ffed 100644 --- a/tools/perf/util/auxtrace.h +++ b/tools/perf/util/auxtrace.h @@ -75,7 +75,6 @@ enum itrace_period_type { * (not fully accurate, since CYC packets are only emitted * together with other events, such as branches) * @branches: whether to synthesize 'branches' events - * (branch misses only for Arm SPE) * @transactions: whether to synthesize events for transactions * @ptwrites: whether to synthesize events for ptwrites * @pwr_events: whether to synthesize power events @@ -650,7 +649,7 @@ bool auxtrace__evsel_is_auxtrace(struct perf_session *session, #define ITRACE_HELP \ " i[period]: synthesize instructions events\n" \ " y[period]: synthesize cycles events (same period as i)\n" \ -" b: synthesize branches events (branch misses for Arm SPE)\n" \ +" b: synthesize branches events\n" \ " c: synthesize branches events (calls only)\n" \ " r: synthesize branches events (returns only)\n" \ " x: synthesize transactions events\n" \ diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index e87b6789eb9e..a4fdf6911ec1 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -375,7 +375,7 @@ static int create_idx_hash(struct evsel *evsel, struct perf_bpf_filter_entry *en pfi = zalloc(sizeof(*pfi)); if (pfi == NULL) { pr_err("Cannot save pinned filter index\n"); - goto err; + return -ENOMEM; } pfi->evsel = evsel; diff --git a/tools/perf/util/bpf-prologue.h b/tools/perf/util/bpf-prologue.h deleted file mode 100644 index 66dcf751ef65..000000000000 --- a/tools/perf/util/bpf-prologue.h +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015, He Kuang <hekuang@huawei.com> - * Copyright (C) 2015, Huawei Inc. - */ -#ifndef __BPF_PROLOGUE_H -#define __BPF_PROLOGUE_H - -struct probe_trace_arg; -struct bpf_insn; - -#define BPF_PROLOGUE_MAX_ARGS 3 -#define BPF_PROLOGUE_START_ARG_REG BPF_REG_3 -#define BPF_PROLOGUE_FETCH_RESULT_REG BPF_REG_2 - -#ifdef HAVE_BPF_PROLOGUE -int bpf__gen_prologue(struct probe_trace_arg *args, int nargs, - struct bpf_insn *new_prog, size_t *new_cnt, - size_t cnt_space); -#else -#include <linux/compiler.h> -#include <errno.h> - -static inline int -bpf__gen_prologue(struct probe_trace_arg *args __maybe_unused, - int nargs __maybe_unused, - struct bpf_insn *new_prog __maybe_unused, - size_t *new_cnt, - size_t cnt_space __maybe_unused) -{ - if (!new_cnt) - return -EINVAL; - *new_cnt = 0; - return -ENOTSUP; -} -#endif -#endif /* __BPF_PROLOGUE_H */ diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index 7a8af60e0f51..73fcafbffc6a 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -394,6 +394,7 @@ static int bperf_check_target(struct evsel *evsel, } static struct perf_cpu_map *all_cpu_map; +static __u32 filter_entry_cnt; static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, struct perf_event_attr_map_entry *entry) @@ -444,12 +445,32 @@ out: return err; } +static int bperf_attach_follower_program(struct bperf_follower_bpf *skel, + enum bperf_filter_type filter_type, + bool inherit) +{ + struct bpf_link *link; + int err = 0; + + if ((filter_type == BPERF_FILTER_PID || + filter_type == BPERF_FILTER_TGID) && inherit) + /* attach all follower bpf progs to enable event inheritance */ + err = bperf_follower_bpf__attach(skel); + else { + link = bpf_program__attach(skel->progs.fexit_XXX); + if (IS_ERR(link)) + err = PTR_ERR(link); + } + + return err; +} + static int bperf__load(struct evsel *evsel, struct target *target) { struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff}; int attr_map_fd, diff_map_fd = -1, err; enum bperf_filter_type filter_type; - __u32 filter_entry_cnt, i; + __u32 i; if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt)) return -1; @@ -529,9 +550,6 @@ static int bperf__load(struct evsel *evsel, struct target *target) /* set up reading map */ bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings, filter_entry_cnt); - /* set up follower filter based on target */ - bpf_map__set_max_entries(evsel->follower_skel->maps.filter, - filter_entry_cnt); err = bperf_follower_bpf__load(evsel->follower_skel); if (err) { pr_err("Failed to load follower skeleton\n"); @@ -543,6 +561,7 @@ static int bperf__load(struct evsel *evsel, struct target *target) for (i = 0; i < filter_entry_cnt; i++) { int filter_map_fd; __u32 key; + struct bperf_filter_value fval = { i, 0 }; if (filter_type == BPERF_FILTER_PID || filter_type == BPERF_FILTER_TGID) @@ -553,12 +572,14 @@ static int bperf__load(struct evsel *evsel, struct target *target) break; filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter); - bpf_map_update_elem(filter_map_fd, &key, &i, BPF_ANY); + bpf_map_update_elem(filter_map_fd, &key, &fval, BPF_ANY); } evsel->follower_skel->bss->type = filter_type; + evsel->follower_skel->bss->inherit = target->inherit; - err = bperf_follower_bpf__attach(evsel->follower_skel); + err = bperf_attach_follower_program(evsel->follower_skel, filter_type, + target->inherit); out: if (err && evsel->bperf_leader_link_fd >= 0) @@ -623,7 +644,7 @@ static int bperf__read(struct evsel *evsel) bperf_sync_counters(evsel); reading_map_fd = bpf_map__fd(skel->maps.accum_readings); - for (i = 0; i < bpf_map__max_entries(skel->maps.accum_readings); i++) { + for (i = 0; i < filter_entry_cnt; i++) { struct perf_cpu entry; __u32 cpu; diff --git a/tools/perf/util/bpf_skel/bperf_follower.bpf.c b/tools/perf/util/bpf_skel/bperf_follower.bpf.c index f193998530d4..0595063139a3 100644 --- a/tools/perf/util/bpf_skel/bperf_follower.bpf.c +++ b/tools/perf/util/bpf_skel/bperf_follower.bpf.c @@ -5,6 +5,8 @@ #include <bpf/bpf_tracing.h> #include "bperf_u.h" +#define MAX_ENTRIES 102400 + struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(__u32)); @@ -22,25 +24,29 @@ struct { struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(key_size, sizeof(__u32)); - __uint(value_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bperf_filter_value)); + __uint(max_entries, MAX_ENTRIES); + __uint(map_flags, BPF_F_NO_PREALLOC); } filter SEC(".maps"); enum bperf_filter_type type = 0; int enabled = 0; +int inherit; SEC("fexit/XXX") int BPF_PROG(fexit_XXX) { struct bpf_perf_event_value *diff_val, *accum_val; __u32 filter_key, zero = 0; - __u32 *accum_key; + __u32 accum_key; + struct bperf_filter_value *fval; if (!enabled) return 0; switch (type) { case BPERF_FILTER_GLOBAL: - accum_key = &zero; + accum_key = zero; goto do_add; case BPERF_FILTER_CPU: filter_key = bpf_get_smp_processor_id(); @@ -49,22 +55,34 @@ int BPF_PROG(fexit_XXX) filter_key = bpf_get_current_pid_tgid() & 0xffffffff; break; case BPERF_FILTER_TGID: - filter_key = bpf_get_current_pid_tgid() >> 32; + /* Use pid as the filter_key to exclude new task counts + * when inherit is disabled. Don't worry about the existing + * children in TGID losing their counts, bpf_counter has + * already added them to the filter map via perf_thread_map + * before this bpf prog runs. + */ + filter_key = inherit ? + bpf_get_current_pid_tgid() >> 32 : + bpf_get_current_pid_tgid() & 0xffffffff; break; default: return 0; } - accum_key = bpf_map_lookup_elem(&filter, &filter_key); - if (!accum_key) + fval = bpf_map_lookup_elem(&filter, &filter_key); + if (!fval) return 0; + accum_key = fval->accum_key; + if (fval->exited) + bpf_map_delete_elem(&filter, &filter_key); + do_add: diff_val = bpf_map_lookup_elem(&diff_readings, &zero); if (!diff_val) return 0; - accum_val = bpf_map_lookup_elem(&accum_readings, accum_key); + accum_val = bpf_map_lookup_elem(&accum_readings, &accum_key); if (!accum_val) return 0; @@ -75,4 +93,70 @@ do_add: return 0; } +/* The program is only used for PID or TGID filter types. */ +SEC("tp_btf/task_newtask") +int BPF_PROG(on_newtask, struct task_struct *task, __u64 clone_flags) +{ + __u32 parent_key, child_key; + struct bperf_filter_value *parent_fval; + struct bperf_filter_value child_fval = { 0 }; + + if (!enabled) + return 0; + + switch (type) { + case BPERF_FILTER_PID: + parent_key = bpf_get_current_pid_tgid() & 0xffffffff; + child_key = task->pid; + break; + case BPERF_FILTER_TGID: + parent_key = bpf_get_current_pid_tgid() >> 32; + child_key = task->tgid; + if (child_key == parent_key) + return 0; + break; + default: + return 0; + } + + /* Check if the current task is one of the target tasks to be counted */ + parent_fval = bpf_map_lookup_elem(&filter, &parent_key); + if (!parent_fval) + return 0; + + /* Start counting for the new task by adding it into filter map, + * inherit the accum key of its parent task so that they can be + * counted together. + */ + child_fval.accum_key = parent_fval->accum_key; + child_fval.exited = 0; + bpf_map_update_elem(&filter, &child_key, &child_fval, BPF_NOEXIST); + + return 0; +} + +/* The program is only used for PID or TGID filter types. */ +SEC("tp_btf/sched_process_exit") +int BPF_PROG(on_exittask, struct task_struct *task) +{ + __u32 pid; + struct bperf_filter_value *fval; + + if (!enabled) + return 0; + + /* Stop counting for this task by removing it from filter map. + * For TGID type, if the pid can be found in the map, it means that + * this pid belongs to the leader task. After the task exits, the + * tgid of its child tasks (if any) will be 1, so the pid can be + * safely removed. + */ + pid = task->pid; + fval = bpf_map_lookup_elem(&filter, &pid); + if (fval) + fval->exited = 1; + + return 0; +} + char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_u.h b/tools/perf/util/bpf_skel/bperf_u.h index 1ce0c2c905c1..4a4a753980be 100644 --- a/tools/perf/util/bpf_skel/bperf_u.h +++ b/tools/perf/util/bpf_skel/bperf_u.h @@ -11,4 +11,9 @@ enum bperf_filter_type { BPERF_FILTER_TGID, }; +struct bperf_filter_value { + __u32 accum_key; + __u8 exited; +}; + #endif /* __BPERF_STAT_U_H */ diff --git a/tools/perf/util/color.c b/tools/perf/util/color.c index bffbdd216a6a..e51f0a676a22 100644 --- a/tools/perf/util/color.c +++ b/tools/perf/util/color.c @@ -93,34 +93,6 @@ int color_fprintf(FILE *fp, const char *color, const char *fmt, ...) return r; } -/* - * This function splits the buffer by newlines and colors the lines individually. - * - * Returns 0 on success. - */ -int color_fwrite_lines(FILE *fp, const char *color, - size_t count, const char *buf) -{ - if (!*color) - return fwrite(buf, count, 1, fp) != 1; - - while (count) { - char *p = memchr(buf, '\n', count); - - if (p != buf && (fputs(color, fp) < 0 || - fwrite(buf, p ? (size_t)(p - buf) : count, 1, fp) != 1 || - fputs(PERF_COLOR_RESET, fp) < 0)) - return -1; - if (!p) - return 0; - if (fputc('\n', fp) < 0) - return -1; - count -= p + 1 - buf; - buf = p + 1; - } - return 0; -} - const char *get_percent_color(double percent) { const char *color = PERF_COLOR_NORMAL; diff --git a/tools/perf/util/color.h b/tools/perf/util/color.h index 01f7bed21c9b..9a7248dbe2d7 100644 --- a/tools/perf/util/color.h +++ b/tools/perf/util/color.h @@ -2,6 +2,7 @@ #ifndef __PERF_COLOR_H #define __PERF_COLOR_H +#include <linux/compiler.h> #include <stdio.h> #include <stdarg.h> @@ -22,6 +23,7 @@ #define MIN_GREEN 0.5 #define MIN_RED 5.0 +#define PERF_COLOR_DELETE_LINE "\033[A\33[2K\r" /* * This variable stores the value of color.ui */ @@ -37,12 +39,11 @@ int perf_config_colorbool(const char *var, const char *value, int stdout_is_tty) int color_vsnprintf(char *bf, size_t size, const char *color, const char *fmt, va_list args); int color_vfprintf(FILE *fp, const char *color, const char *fmt, va_list args); -int color_fprintf(FILE *fp, const char *color, const char *fmt, ...); -int color_snprintf(char *bf, size_t size, const char *color, const char *fmt, ...); -int color_fwrite_lines(FILE *fp, const char *color, size_t count, const char *buf); +int color_fprintf(FILE *fp, const char *color, const char *fmt, ...) __printf(3, 4); +int color_snprintf(char *bf, size_t size, const char *color, const char *fmt, ...) __printf(4, 5); int value_color_snprintf(char *bf, size_t size, const char *fmt, double value); -int percent_color_snprintf(char *bf, size_t size, const char *fmt, ...); -int percent_color_len_snprintf(char *bf, size_t size, const char *fmt, ...); +int percent_color_snprintf(char *bf, size_t size, const char *fmt, ...) __printf(3, 4); +int percent_color_len_snprintf(char *bf, size_t size, const char *fmt, ...) __printf(3, 4); int percent_color_fprintf(FILE *fp, const char *fmt, double percent); const char *get_percent_color(double percent); diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c index 7a650de0db83..68f9407ca74b 100644 --- a/tools/perf/util/config.c +++ b/tools/perf/util/config.c @@ -912,6 +912,7 @@ void set_buildid_dir(const char *dir) struct perf_config_scan_data { const char *name; const char *fmt; + const char *value; va_list args; int ret; }; @@ -939,3 +940,24 @@ int perf_config_scan(const char *name, const char *fmt, ...) return d.ret; } + +static int perf_config_get_cb(const char *var, const char *value, void *data) +{ + struct perf_config_scan_data *d = data; + + if (!strcmp(var, d->name)) + d->value = value; + + return 0; +} + +const char *perf_config_get(const char *name) +{ + struct perf_config_scan_data d = { + .name = name, + .value = NULL, + }; + + perf_config(perf_config_get_cb, &d); + return d.value; +} diff --git a/tools/perf/util/config.h b/tools/perf/util/config.h index 2e5e808928a5..9971313d61c1 100644 --- a/tools/perf/util/config.h +++ b/tools/perf/util/config.h @@ -30,6 +30,7 @@ typedef int (*config_fn_t)(const char *, const char *, void *); int perf_default_config(const char *, const char *, void *); int perf_config(config_fn_t fn, void *); int perf_config_scan(const char *name, const char *fmt, ...) __scanf(2, 3); +const char *perf_config_get(const char *name); int perf_config_set(struct perf_config_set *set, config_fn_t fn, void *data); int perf_config_int(int *dest, const char *, const char *); diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index b78ef0262135..b85a8837bddc 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -685,9 +685,14 @@ cs_etm_decoder__create_etm_decoder(struct cs_etm_decoder_params *d_params, } if (d_params->operation == CS_ETM_OPERATION_DECODE) { + int decode_flags = OCSD_CREATE_FLG_FULL_DECODER; +#ifdef OCSD_OPFLG_N_UNCOND_DIR_BR_CHK + decode_flags |= OCSD_OPFLG_N_UNCOND_DIR_BR_CHK | OCSD_OPFLG_CHK_RANGE_CONTINUE | + ETM4_OPFLG_PKTDEC_AA64_OPCODE_CHK; +#endif if (ocsd_dt_create_decoder(decoder->dcd_tree, decoder->decoder_name, - OCSD_CREATE_FLG_FULL_DECODER, + decode_flags, trace_config, &csid)) return -1; diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 40f047baef81..0bf9e5c27b59 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -2490,12 +2490,6 @@ static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) /* Ignore return value */ cs_etm__process_traceid_queue(etmq, tidq); - - /* - * Generate an instruction sample with the remaining - * branchstack entries. - */ - cs_etm__flush(etmq, tidq); } } @@ -2638,7 +2632,7 @@ static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm) while (1) { if (!etm->heap.heap_cnt) - goto out; + break; /* Take the entry at the top of the min heap */ cs_queue_nr = etm->heap.heap_array[0].queue_nr; @@ -2721,6 +2715,23 @@ refetch: ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); } + for (i = 0; i < etm->queues.nr_queues; i++) { + struct int_node *inode; + + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + intlist__for_each_entry(inode, etmq->traceid_queues_list) { + int idx = (int)(intptr_t)inode->priv; + + /* Flush any remaining branch stack entries */ + tidq = etmq->traceid_queues[idx]; + ret = cs_etm__end_block(etmq, tidq); + if (ret) + return ret; + } + } out: return ret; } diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c index 021e9b1d5cc5..f0599c61fab4 100644 --- a/tools/perf/util/data-convert-bt.c +++ b/tools/perf/util/data-convert-bt.c @@ -36,7 +36,7 @@ #include "util/sample.h" #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif #define pr_N(n, fmt, ...) \ diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index 20bfb0884e9e..8304cd2d4a9c 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -28,7 +28,7 @@ #include "util/tool.h" #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif struct convert_json { diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c index d633d15329fa..995f6bb05b5f 100644 --- a/tools/perf/util/debug.c +++ b/tools/perf/util/debug.c @@ -27,7 +27,7 @@ #include <linux/ctype.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #else #define LIBTRACEEVENT_VERSION 0 #endif diff --git a/tools/perf/util/debuginfo.h b/tools/perf/util/debuginfo.h index ad6422c3f8ca..a52d69932815 100644 --- a/tools/perf/util/debuginfo.h +++ b/tools/perf/util/debuginfo.h @@ -5,7 +5,7 @@ #include <errno.h> #include <linux/compiler.h> -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT #include "dwarf-aux.h" @@ -25,7 +25,7 @@ void debuginfo__delete(struct debuginfo *dbg); int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, bool adjust_offset); -#else /* HAVE_DWARF_SUPPORT */ +#else /* HAVE_LIBDW_SUPPORT */ /* dummy debug information structure */ struct debuginfo { @@ -49,7 +49,7 @@ static inline int debuginfo__get_text_offset(struct debuginfo *dbg __maybe_unuse return -EINVAL; } -#endif /* HAVE_DWARF_SUPPORT */ +#endif /* HAVE_LIBDW_SUPPORT */ #ifdef HAVE_DEBUGINFOD_SUPPORT int get_source_from_debuginfod(const char *raw_path, const char *sbuild_id, diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index f05ba7739c1e..41a2b08670dc 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -18,6 +18,7 @@ #include "disasm.h" #include "disasm_bpf.h" #include "dso.h" +#include "dwarf-regs.h" #include "env.h" #include "evsel.h" #include "map.h" @@ -151,14 +152,14 @@ static struct arch architectures[] = { .memory_ref_char = '(', .imm_char = '$', }, -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT .update_insn_state = update_insn_state_x86, #endif }, { .name = "powerpc", .init = powerpc__annotate_init, -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT .update_insn_state = update_insn_state_powerpc, #endif }, @@ -1423,6 +1424,15 @@ err: } #endif +#if !defined(HAVE_LIBCAPSTONE_SUPPORT) || !defined(HAVE_LIBLLVM_SUPPORT) +static void symbol__disassembler_missing(const char *disassembler, const char *filename, + struct symbol *sym) +{ + pr_debug("The %s disassembler isn't linked in for %s in %s\n", + disassembler, sym->name, filename); +} +#endif + #ifdef HAVE_LIBCAPSTONE_SUPPORT static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, struct annotate_args *args, u64 addr) @@ -1573,7 +1583,7 @@ static int symbol__disassemble_capstone_powerpc(char *filename, struct symbol *s dl = disasm_line__new(args); if (dl == NULL) - goto err; + break; annotation_line__add(&dl->al, ¬es->src->source); @@ -1603,18 +1613,6 @@ out: err: if (fd >= 0) close(fd); - if (needs_cs_close) { - struct disasm_line *tmp; - - /* - * It probably failed in the middle of the above loop. - * Release any resources it might add. - */ - list_for_each_entry_safe(dl, tmp, ¬es->src->source, al.node) { - list_del(&dl->al.node); - free(dl); - } - } count = -1; goto out; } @@ -1627,12 +1625,12 @@ static int symbol__disassemble_capstone(char *filename, struct symbol *sym, u64 start = map__rip_2objdump(map, sym->start); u64 len; u64 offset; - int i, count; + int i, count, free_count; bool is_64bit = false; bool needs_cs_close = false; u8 *buf = NULL; csh handle; - cs_insn *insn; + cs_insn *insn = NULL; char disasm_buf[512]; struct disasm_line *dl; @@ -1664,7 +1662,7 @@ static int symbol__disassemble_capstone(char *filename, struct symbol *sym, needs_cs_close = true; - count = cs_disasm(handle, buf, len, start, len, &insn); + free_count = count = cs_disasm(handle, buf, len, start, len, &insn); for (i = 0, offset = 0; i < count; i++) { int printed; @@ -1702,8 +1700,11 @@ static int symbol__disassemble_capstone(char *filename, struct symbol *sym, } out: - if (needs_cs_close) + if (needs_cs_close) { cs_close(&handle); + if (free_count > 0) + cs_free(insn, free_count); + } free(buf); return count < 0 ? count : 0; @@ -1717,13 +1718,27 @@ err: */ list_for_each_entry_safe(dl, tmp, ¬es->src->source, al.node) { list_del(&dl->al.node); - free(dl); + disasm_line__free(dl); } } count = -1; goto out; } -#endif +#else // HAVE_LIBCAPSTONE_SUPPORT +static int symbol__disassemble_capstone(char *filename, struct symbol *sym, + struct annotate_args *args __maybe_unused) +{ + symbol__disassembler_missing("capstone", filename, sym); + return -1; +} + +static int symbol__disassemble_capstone_powerpc(char *filename, struct symbol *sym, + struct annotate_args *args __maybe_unused) +{ + symbol__disassembler_missing("capstone powerpc", filename, sym); + return -1; +} +#endif // HAVE_LIBCAPSTONE_SUPPORT static int symbol__disassemble_raw(char *filename, struct symbol *sym, struct annotate_args *args) @@ -1782,7 +1797,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, sprintf(args->line, "%x", line[i]); dl = disasm_line__new(args); if (dl == NULL) - goto err; + break; annotation_line__add(&dl->al, ¬es->src->source); offset += 4; @@ -1991,7 +2006,14 @@ err: free(line_storage); return ret; } -#endif +#else // HAVE_LIBLLVM_SUPPORT +static int symbol__disassemble_llvm(char *filename, struct symbol *sym, + struct annotate_args *args __maybe_unused) +{ + symbol__disassembler_missing("LLVM", filename, sym); + return -1; +} +#endif // HAVE_LIBLLVM_SUPPORT /* * Possibly create a new version of line with tabs expanded. Returns the @@ -2053,17 +2075,14 @@ static char *expand_tabs(char *line, char **storage, size_t *storage_len) return new_line; } -int symbol__disassemble(struct symbol *sym, struct annotate_args *args) +static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, + struct annotate_args *args) { struct annotation_options *opts = &annotate_opts; struct map *map = args->ms.map; struct dso *dso = map__dso(map); char *command; FILE *file; - char symfs_filename[PATH_MAX]; - struct kcore_extract kce; - bool delete_extract = false; - bool decomp = false; int lineno = 0; char *fileloc = NULL; int nline; @@ -2078,77 +2097,7 @@ int symbol__disassemble(struct symbol *sym, struct annotate_args *args) NULL, }; struct child_process objdump_process; - int err = dso__disassemble_filename(dso, symfs_filename, sizeof(symfs_filename)); - - if (err) - return err; - - pr_debug("%s: filename=%s, sym=%s, start=%#" PRIx64 ", end=%#" PRIx64 "\n", __func__, - symfs_filename, sym->name, map__unmap_ip(map, sym->start), - map__unmap_ip(map, sym->end)); - - pr_debug("annotating [%p] %30s : [%p] %30s\n", - dso, dso__long_name(dso), sym, sym->name); - - if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) { - return symbol__disassemble_bpf(sym, args); - } else if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) { - return symbol__disassemble_bpf_image(sym, args); - } else if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) { - return -1; - } else if (dso__is_kcore(dso)) { - kce.kcore_filename = symfs_filename; - kce.addr = map__rip_2objdump(map, sym->start); - kce.offs = sym->start; - kce.len = sym->end - sym->start; - if (!kcore_extract__create(&kce)) { - delete_extract = true; - strlcpy(symfs_filename, kce.extract_filename, - sizeof(symfs_filename)); - } - } else if (dso__needs_decompress(dso)) { - char tmp[KMOD_DECOMP_LEN]; - - if (dso__decompress_kmodule_path(dso, symfs_filename, - tmp, sizeof(tmp)) < 0) - return -1; - - decomp = true; - strcpy(symfs_filename, tmp); - } - - /* - * For powerpc data type profiling, use the dso__data_read_offset - * to read raw instruction directly and interpret the binary code - * to understand instructions and register fields. For sort keys as - * type and typeoff, disassemble to mnemonic notation is - * not required in case of powerpc. - */ - if (arch__is(args->arch, "powerpc")) { - extern const char *sort_order; - - if (sort_order && !strstr(sort_order, "sym")) { - err = symbol__disassemble_raw(symfs_filename, sym, args); - if (err == 0) - goto out_remove_tmp; -#ifdef HAVE_LIBCAPSTONE_SUPPORT - err = symbol__disassemble_capstone_powerpc(symfs_filename, sym, args); - if (err == 0) - goto out_remove_tmp; -#endif - } - } - -#ifdef HAVE_LIBLLVM_SUPPORT - err = symbol__disassemble_llvm(symfs_filename, sym, args); - if (err == 0) - goto out_remove_tmp; -#endif -#ifdef HAVE_LIBCAPSTONE_SUPPORT - err = symbol__disassemble_capstone(symfs_filename, sym, args); - if (err == 0) - goto out_remove_tmp; -#endif + int err; err = asprintf(&command, "%s %s%s --start-address=0x%016" PRIx64 @@ -2171,13 +2120,13 @@ int symbol__disassemble(struct symbol *sym, struct annotate_args *args) if (err < 0) { pr_err("Failure allocating memory for the command to run\n"); - goto out_remove_tmp; + return err; } pr_debug("Executing: %s\n", command); objdump_argv[2] = command; - objdump_argv[4] = symfs_filename; + objdump_argv[4] = filename; /* Create a pipe to read from for stdout */ memset(&objdump_process, 0, sizeof(objdump_process)); @@ -2215,8 +2164,8 @@ int symbol__disassemble(struct symbol *sym, struct annotate_args *args) break; /* Skip lines containing "filename:" */ - match = strstr(line, symfs_filename); - if (match && match[strlen(symfs_filename)] == ':') + match = strstr(line, filename); + if (match && match[strlen(filename)] == ':') continue; expanded_line = strim(line); @@ -2261,7 +2210,150 @@ out_close_stdout: out_free_command: free(command); + return err; +} + +static int annotation_options__init_disassemblers(struct annotation_options *options) +{ + char *disassembler; + + if (options->disassemblers_str == NULL) { + const char *default_disassemblers_str = +#ifdef HAVE_LIBLLVM_SUPPORT + "llvm," +#endif +#ifdef HAVE_LIBCAPSTONE_SUPPORT + "capstone," +#endif + "objdump"; + + options->disassemblers_str = strdup(default_disassemblers_str); + if (!options->disassemblers_str) + goto out_enomem; + } + + disassembler = strdup(options->disassemblers_str); + if (disassembler == NULL) + goto out_enomem; + + while (1) { + char *comma = strchr(disassembler, ','); + + if (comma != NULL) + *comma = '\0'; + + options->disassemblers[options->nr_disassemblers++] = strim(disassembler); + + if (comma == NULL) + break; + + disassembler = comma + 1; + + if (options->nr_disassemblers >= MAX_DISASSEMBLERS) { + pr_debug("annotate.disassemblers can have at most %d entries, ignoring \"%s\"\n", + MAX_DISASSEMBLERS, disassembler); + break; + } + } + + return 0; + +out_enomem: + pr_err("Not enough memory for annotate.disassemblers\n"); + return -1; +} + +int symbol__disassemble(struct symbol *sym, struct annotate_args *args) +{ + struct annotation_options *options = args->options; + struct map *map = args->ms.map; + struct dso *dso = map__dso(map); + char symfs_filename[PATH_MAX]; + bool delete_extract = false; + struct kcore_extract kce; + const char *disassembler; + bool decomp = false; + int err = dso__disassemble_filename(dso, symfs_filename, sizeof(symfs_filename)); + + if (err) + return err; + + pr_debug("%s: filename=%s, sym=%s, start=%#" PRIx64 ", end=%#" PRIx64 "\n", __func__, + symfs_filename, sym->name, map__unmap_ip(map, sym->start), + map__unmap_ip(map, sym->end)); + + pr_debug("annotating [%p] %30s : [%p] %30s\n", dso, dso__long_name(dso), sym, sym->name); + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) { + return symbol__disassemble_bpf(sym, args); + } else if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) { + return symbol__disassemble_bpf_image(sym, args); + } else if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) { + return -1; + } else if (dso__is_kcore(dso)) { + kce.addr = map__rip_2objdump(map, sym->start); + kce.kcore_filename = symfs_filename; + kce.len = sym->end - sym->start; + kce.offs = sym->start; + + if (!kcore_extract__create(&kce)) { + delete_extract = true; + strlcpy(symfs_filename, kce.extract_filename, sizeof(symfs_filename)); + } + } else if (dso__needs_decompress(dso)) { + char tmp[KMOD_DECOMP_LEN]; + + if (dso__decompress_kmodule_path(dso, symfs_filename, tmp, sizeof(tmp)) < 0) + return -1; + + decomp = true; + strcpy(symfs_filename, tmp); + } + + /* + * For powerpc data type profiling, use the dso__data_read_offset to + * read raw instruction directly and interpret the binary code to + * understand instructions and register fields. For sort keys as type + * and typeoff, disassemble to mnemonic notation is not required in + * case of powerpc. + */ + if (arch__is(args->arch, "powerpc")) { + extern const char *sort_order; + + if (sort_order && !strstr(sort_order, "sym")) { + err = symbol__disassemble_raw(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + + err = symbol__disassemble_capstone_powerpc(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + } + } + + err = annotation_options__init_disassemblers(options); + if (err) + goto out_remove_tmp; + + err = -1; + + for (int i = 0; i < options->nr_disassemblers && err != 0; ++i) { + disassembler = options->disassemblers[i]; + + if (!strcmp(disassembler, "llvm")) + err = symbol__disassemble_llvm(symfs_filename, sym, args); + else if (!strcmp(disassembler, "capstone")) + err = symbol__disassemble_capstone(symfs_filename, sym, args); + else if (!strcmp(disassembler, "objdump")) + err = symbol__disassemble_objdump(symfs_filename, sym, args); + else + pr_debug("Unknown disassembler %s, skipping...\n", disassembler); + } + + if (err == 0) { + pr_debug("Disassembled with %s\nannotate.disassemblers=%s\n", + disassembler, options->disassemblers_str); + } out_remove_tmp: if (decomp) unlink(symfs_filename); diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h index f56beedeb9da..c135db2416b5 100644 --- a/tools/perf/util/disasm.h +++ b/tools/perf/util/disasm.h @@ -4,7 +4,7 @@ #include "map_symbol.h" -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT #include "dwarf-aux.h" #endif @@ -39,11 +39,15 @@ struct arch { char memory_ref_char; char imm_char; } objdump; -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT void (*update_insn_state)(struct type_state *state, struct data_loc_info *dloc, Dwarf_Die *cu_die, struct disasm_line *dl); #endif + /** @e_machine: ELF machine associated with arch. */ + unsigned int e_machine; + /** @e_flags: Optional ELF flags associated with arch. */ + unsigned int e_flags; }; struct ins { diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c index 92eb9c8dc3e5..559c953ca172 100644 --- a/tools/perf/util/dwarf-aux.c +++ b/tools/perf/util/dwarf-aux.c @@ -1182,7 +1182,6 @@ int die_get_varname(Dwarf_Die *vr_die, struct strbuf *buf) return ret < 0 ? ret : strbuf_addf(buf, "\t%s", dwarf_diename(vr_die)); } -#if defined(HAVE_DWARF_GETLOCATIONS_SUPPORT) || defined(HAVE_DWARF_CFI_SUPPORT) static int reg_from_dwarf_op(Dwarf_Op *op) { switch (op->atom) { @@ -1245,9 +1244,7 @@ static bool check_allowed_ops(Dwarf_Op *ops, size_t nops) } return true; } -#endif /* HAVE_DWARF_GETLOCATIONS_SUPPORT || HAVE_DWARF_CFI_SUPPORT */ -#ifdef HAVE_DWARF_GETLOCATIONS_SUPPORT /** * die_get_var_innermost_scope - Get innermost scope range of given variable DIE * @sp_die: a subprogram DIE @@ -1697,9 +1694,7 @@ void die_collect_global_vars(Dwarf_Die *cu_die, struct die_var_type **var_types) die_find_child(cu_die, __die_collect_global_vars_cb, (void *)var_types, &die_mem); } -#endif /* HAVE_DWARF_GETLOCATIONS_SUPPORT */ -#ifdef HAVE_DWARF_CFI_SUPPORT /** * die_get_cfa - Get frame base information * @dwarf: a Dwarf info @@ -1732,7 +1727,6 @@ int die_get_cfa(Dwarf *dwarf, u64 pc, int *preg, int *poffset) } return -1; } -#endif /* HAVE_DWARF_CFI_SUPPORT */ /* * die_has_loclist - Check if DW_AT_location of @vr_die is a location list diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h index bd7505812569..892c8c5c23fc 100644 --- a/tools/perf/util/dwarf-aux.h +++ b/tools/perf/util/dwarf-aux.h @@ -9,7 +9,6 @@ #include <elfutils/libdw.h> #include <elfutils/libdwfl.h> #include <elfutils/version.h> -#include <errno.h> struct strbuf; @@ -157,8 +156,6 @@ Dwarf_Die *die_get_member_type(Dwarf_Die *type_die, int offset, Dwarf_Die *die_m /* Return type info where the pointer and offset point to */ Dwarf_Die *die_deref_ptr_type(Dwarf_Die *ptr_die, int offset, Dwarf_Die *die_mem); -#ifdef HAVE_DWARF_GETLOCATIONS_SUPPORT - /* Get byte offset range of given variable DIE */ int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf); @@ -177,58 +174,7 @@ void die_collect_vars(Dwarf_Die *sc_die, struct die_var_type **var_types); /* Save all global variables in this CU */ void die_collect_global_vars(Dwarf_Die *cu_die, struct die_var_type **var_types); -#else /* HAVE_DWARF_GETLOCATIONS_SUPPORT */ - -static inline int die_get_var_range(Dwarf_Die *sp_die __maybe_unused, - Dwarf_Die *vr_die __maybe_unused, - struct strbuf *buf __maybe_unused) -{ - return -ENOTSUP; -} - -static inline Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die __maybe_unused, - Dwarf_Addr pc __maybe_unused, - int reg __maybe_unused, - int *poffset __maybe_unused, - bool is_fbreg __maybe_unused, - Dwarf_Die *die_mem __maybe_unused) -{ - return NULL; -} - -static inline Dwarf_Die *die_find_variable_by_addr(Dwarf_Die *sc_die __maybe_unused, - Dwarf_Addr addr __maybe_unused, - Dwarf_Die *die_mem __maybe_unused, - int *offset __maybe_unused) -{ - return NULL; -} - -static inline void die_collect_vars(Dwarf_Die *sc_die __maybe_unused, - struct die_var_type **var_types __maybe_unused) -{ -} - -static inline void die_collect_global_vars(Dwarf_Die *cu_die __maybe_unused, - struct die_var_type **var_types __maybe_unused) -{ -} - -#endif /* HAVE_DWARF_GETLOCATIONS_SUPPORT */ - -#ifdef HAVE_DWARF_CFI_SUPPORT - /* Get the frame base information from CFA */ int die_get_cfa(Dwarf *dwarf, u64 pc, int *preg, int *poffset); -#else /* HAVE_DWARF_CFI_SUPPORT */ - -static inline int die_get_cfa(Dwarf *dwarf __maybe_unused, u64 pc __maybe_unused, - int *preg __maybe_unused, int *poffset __maybe_unused) -{ - return -1; -} - -#endif /* HAVE_DWARF_CFI_SUPPORT */ - #endif /* _DWARF_AUX_H */ diff --git a/tools/perf/arch/csky/util/dwarf-regs.c b/tools/perf/util/dwarf-regs-csky.c index ca86ecaeacbb..d38ef1f07f3e 100644 --- a/tools/perf/arch/csky/util/dwarf-regs.c +++ b/tools/perf/util/dwarf-regs-csky.c @@ -5,9 +5,8 @@ #include <stddef.h> #include <dwarf-regs.h> -#if defined(__CSKYABIV2__) -#define CSKY_MAX_REGS 73 -const char *csky_dwarf_regs_table[CSKY_MAX_REGS] = { +#define CSKY_ABIV2_MAX_REGS 73 +const char *csky_dwarf_regs_table_abiv2[CSKY_ABIV2_MAX_REGS] = { /* r0 ~ r8 */ "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", "%regs2", "%regs3", /* r9 ~ r15 */ @@ -26,9 +25,9 @@ const char *csky_dwarf_regs_table[CSKY_MAX_REGS] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "%epc", }; -#else -#define CSKY_MAX_REGS 57 -const char *csky_dwarf_regs_table[CSKY_MAX_REGS] = { + +#define CSKY_ABIV1_MAX_REGS 57 +const char *csky_dwarf_regs_table_abiv1[CSKY_ABIV1_MAX_REGS] = { /* r0 ~ r8 */ "%sp", "%regs9", "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", /* r9 ~ r15 */ @@ -41,9 +40,11 @@ const char *csky_dwarf_regs_table[CSKY_MAX_REGS] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "%epc", }; -#endif -const char *get_arch_regstr(unsigned int n) +const char *get_csky_regstr(unsigned int n, unsigned int flags) { - return (n < CSKY_MAX_REGS) ? csky_dwarf_regs_table[n] : NULL; + if (flags & EF_CSKY_ABIV2) + return (n < CSKY_ABIV2_MAX_REGS) ? csky_dwarf_regs_table_abiv2[n] : NULL; + + return (n < CSKY_ABIV1_MAX_REGS) ? csky_dwarf_regs_table_abiv1[n] : NULL; } diff --git a/tools/perf/util/dwarf-regs-powerpc.c b/tools/perf/util/dwarf-regs-powerpc.c new file mode 100644 index 000000000000..caf77a234c78 --- /dev/null +++ b/tools/perf/util/dwarf-regs-powerpc.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Mapping of DWARF debug register numbers into register names. + * + * Copyright (C) 2010 Ian Munsie, IBM Corporation. + */ + +#include <dwarf-regs.h> + +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define PPC_RA(a) (((a) >> 16) & 0x1f) +#define PPC_RT(t) (((t) >> 21) & 0x1f) +#define PPC_RB(b) (((b) >> 11) & 0x1f) +#define PPC_D(D) ((D) & 0xfffe) +#define PPC_DS(DS) ((DS) & 0xfffc) +#define OP_LD 58 +#define OP_STD 62 + +static int get_source_reg(u32 raw_insn) +{ + return PPC_RA(raw_insn); +} + +static int get_target_reg(u32 raw_insn) +{ + return PPC_RT(raw_insn); +} + +static int get_offset_opcode(u32 raw_insn) +{ + int opcode = PPC_OP(raw_insn); + + /* DS- form */ + if ((opcode == OP_LD) || (opcode == OP_STD)) + return PPC_DS(raw_insn); + else + return PPC_D(raw_insn); +} + +/* + * Fills the required fields for op_loc depending on if it + * is a source or target. + * D form: ins RT,D(RA) -> src_reg1 = RA, offset = D, dst_reg1 = RT + * DS form: ins RT,DS(RA) -> src_reg1 = RA, offset = DS, dst_reg1 = RT + * X form: ins RT,RA,RB -> src_reg1 = RA, src_reg2 = RB, dst_reg1 = RT + */ +void get_powerpc_regs(u32 raw_insn, int is_source, + struct annotated_op_loc *op_loc) +{ + if (is_source) + op_loc->reg1 = get_source_reg(raw_insn); + else + op_loc->reg1 = get_target_reg(raw_insn); + + if (op_loc->multi_regs) + op_loc->reg2 = PPC_RB(raw_insn); + + /* TODO: Implement offset handling for X Form */ + if ((op_loc->mem_ref) && (PPC_OP(raw_insn) != 31)) + op_loc->offset = get_offset_opcode(raw_insn); +} diff --git a/tools/perf/util/dwarf-regs-x86.c b/tools/perf/util/dwarf-regs-x86.c new file mode 100644 index 000000000000..7a55c65e8da6 --- /dev/null +++ b/tools/perf/util/dwarf-regs-x86.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. + * Extracted from probe-finder.c + * + * Written by Masami Hiramatsu <mhiramat@redhat.com> + */ + +#include <errno.h> /* for EINVAL */ +#include <string.h> /* for strcmp */ +#include <linux/kernel.h> /* for ARRAY_SIZE */ +#include <dwarf-regs.h> + +struct dwarf_regs_idx { + const char *name; + int idx; +}; + +static const struct dwarf_regs_idx x86_regidx_table[] = { + { "rax", 0 }, { "eax", 0 }, { "ax", 0 }, { "al", 0 }, + { "rdx", 1 }, { "edx", 1 }, { "dx", 1 }, { "dl", 1 }, + { "rcx", 2 }, { "ecx", 2 }, { "cx", 2 }, { "cl", 2 }, + { "rbx", 3 }, { "edx", 3 }, { "bx", 3 }, { "bl", 3 }, + { "rsi", 4 }, { "esi", 4 }, { "si", 4 }, { "sil", 4 }, + { "rdi", 5 }, { "edi", 5 }, { "di", 5 }, { "dil", 5 }, + { "rbp", 6 }, { "ebp", 6 }, { "bp", 6 }, { "bpl", 6 }, + { "rsp", 7 }, { "esp", 7 }, { "sp", 7 }, { "spl", 7 }, + { "r8", 8 }, { "r8d", 8 }, { "r8w", 8 }, { "r8b", 8 }, + { "r9", 9 }, { "r9d", 9 }, { "r9w", 9 }, { "r9b", 9 }, + { "r10", 10 }, { "r10d", 10 }, { "r10w", 10 }, { "r10b", 10 }, + { "r11", 11 }, { "r11d", 11 }, { "r11w", 11 }, { "r11b", 11 }, + { "r12", 12 }, { "r12d", 12 }, { "r12w", 12 }, { "r12b", 12 }, + { "r13", 13 }, { "r13d", 13 }, { "r13w", 13 }, { "r13b", 13 }, + { "r14", 14 }, { "r14d", 14 }, { "r14w", 14 }, { "r14b", 14 }, + { "r15", 15 }, { "r15d", 15 }, { "r15w", 15 }, { "r15b", 15 }, + { "rip", DWARF_REG_PC }, +}; + +int get_x86_regnum(const char *name) +{ + unsigned int i; + + if (*name != '%') + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(x86_regidx_table); i++) + if (!strcmp(x86_regidx_table[i].name, name + 1)) + return x86_regidx_table[i].idx; + return -ENOENT; +} diff --git a/tools/perf/util/dwarf-regs.c b/tools/perf/util/dwarf-regs.c index 5b7f86c0063f..28a1cfdf26d4 100644 --- a/tools/perf/util/dwarf-regs.c +++ b/tools/perf/util/dwarf-regs.c @@ -13,14 +13,6 @@ #include <errno.h> #include <linux/kernel.h> -#ifndef EM_AARCH64 -#define EM_AARCH64 183 /* ARM 64 bit */ -#endif - -#ifndef EM_LOONGARCH -#define EM_LOONGARCH 258 /* LoongArch */ -#endif - /* Define const char * {arch}_register_tbl[] */ #define DEFINE_DWARF_REGSTR_TABLE #include "../arch/x86/include/dwarf-regs-table.h" @@ -28,6 +20,7 @@ #include "../arch/arm64/include/dwarf-regs-table.h" #include "../arch/sh/include/dwarf-regs-table.h" #include "../arch/powerpc/include/dwarf-regs-table.h" +#include "../arch/riscv/include/dwarf-regs-table.h" #include "../arch/s390/include/dwarf-regs-table.h" #include "../arch/sparc/include/dwarf-regs-table.h" #include "../arch/xtensa/include/dwarf-regs-table.h" @@ -37,11 +30,13 @@ #define __get_dwarf_regstr(tbl, n) (((n) < ARRAY_SIZE(tbl)) ? (tbl)[(n)] : NULL) /* Return architecture dependent register string (for kprobe-tracer) */ -const char *get_dwarf_regstr(unsigned int n, unsigned int machine) +const char *get_dwarf_regstr(unsigned int n, unsigned int machine, unsigned int flags) { + if (machine == EM_NONE) { + /* Generic arch - use host arch */ + machine = EM_HOST; + } switch (machine) { - case EM_NONE: /* Generic arch - use host arch */ - return get_arch_regstr(n); case EM_386: return __get_dwarf_regstr(x86_32_regstr_tbl, n); case EM_X86_64: @@ -50,6 +45,8 @@ const char *get_dwarf_regstr(unsigned int n, unsigned int machine) return __get_dwarf_regstr(arm_regstr_tbl, n); case EM_AARCH64: return __get_dwarf_regstr(aarch64_regstr_tbl, n); + case EM_CSKY: + return get_csky_regstr(n, flags); case EM_SH: return __get_dwarf_regstr(sh_regstr_tbl, n); case EM_S390: @@ -57,6 +54,8 @@ const char *get_dwarf_regstr(unsigned int n, unsigned int machine) case EM_PPC: case EM_PPC64: return __get_dwarf_regstr(powerpc_regstr_tbl, n); + case EM_RISCV: + return __get_dwarf_regstr(riscv_regstr_tbl, n); case EM_SPARC: case EM_SPARCV9: return __get_dwarf_regstr(sparc_regstr_tbl, n); @@ -72,13 +71,15 @@ const char *get_dwarf_regstr(unsigned int n, unsigned int machine) return NULL; } +#if EM_HOST != EM_X86_64 && EM_HOST != EM_386 __weak int get_arch_regnum(const char *name __maybe_unused) { return -ENOTSUP; } +#endif /* Return DWARF register number from architecture register name */ -int get_dwarf_regnum(const char *name, unsigned int machine) +int get_dwarf_regnum(const char *name, unsigned int machine, unsigned int flags __maybe_unused) { char *regname = strdup(name); int reg = -1; @@ -92,10 +93,21 @@ int get_dwarf_regnum(const char *name, unsigned int machine) if (p) *p = '\0'; + if (machine == EM_NONE) { + /* Generic arch - use host arch */ + machine = EM_HOST; + } switch (machine) { - case EM_NONE: /* Generic arch - use host arch */ +#if EM_HOST != EM_X86_64 && EM_HOST != EM_386 + case EM_HOST: reg = get_arch_regnum(regname); break; +#endif + case EM_X86_64: + fallthrough; + case EM_386: + reg = get_x86_regnum(regname); + break; default: pr_err("ELF MACHINE %x is not supported.\n", machine); } diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index 1edbccfc3281..e2843ca2edd9 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -5,12 +5,14 @@ #include "util/header.h" #include "linux/compiler.h" #include <linux/ctype.h> +#include <linux/string.h> #include <linux/zalloc.h> #include "cgroup.h" #include <errno.h> #include <sys/utsname.h> #include <stdlib.h> #include <string.h> +#include "pmu.h" #include "pmus.h" #include "strbuf.h" #include "trace/beauty/beauty.h" @@ -372,7 +374,8 @@ error: int perf_env__read_cpuid(struct perf_env *env) { char cpuid[128]; - int err = get_cpuid(cpuid, sizeof(cpuid)); + struct perf_cpu cpu = {-1}; + int err = get_cpuid(cpuid, sizeof(cpuid), cpu); if (err) return err; @@ -639,3 +642,25 @@ void perf_env__find_br_cntr_info(struct perf_env *env, env->pmu_caps->br_cntr_width; } } + +bool perf_env__is_x86_amd_cpu(struct perf_env *env) +{ + static int is_amd; /* 0: Uninitialized, 1: Yes, -1: No */ + + if (is_amd == 0) + is_amd = env->cpuid && strstarts(env->cpuid, "AuthenticAMD") ? 1 : -1; + + return is_amd >= 1 ? true : false; +} + +bool x86__is_amd_cpu(void) +{ + struct perf_env env = { .total_mem = 0, }; + bool is_amd; + + perf_env__cpuid(&env); + is_amd = perf_env__is_x86_amd_cpu(&env); + perf_env__exit(&env); + + return is_amd; +} diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h index 51b36c36019b..ae604c4edbb7 100644 --- a/tools/perf/util/env.h +++ b/tools/perf/util/env.h @@ -195,4 +195,8 @@ bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name); void perf_env__find_br_cntr_info(struct perf_env *env, unsigned int *nr, unsigned int *width); + +bool x86__is_amd_cpu(void); +bool perf_env__is_x86_amd_cpu(struct perf_env *env); + #endif /* __PERF_ENV_H */ diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h index f8742e6230a5..2744c54f404e 100644 --- a/tools/perf/util/event.h +++ b/tools/perf/util/event.h @@ -66,6 +66,7 @@ enum { PERF_IP_FLAG_VMEXIT = 1ULL << 12, PERF_IP_FLAG_INTR_DISABLE = 1ULL << 13, PERF_IP_FLAG_INTR_TOGGLE = 1ULL << 14, + PERF_IP_FLAG_BRANCH_MISS = 1ULL << 15, }; #define PERF_IP_FLAG_CHARS "bcrosyiABExghDt" diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index f14b7e6ff1dc..f0dd174e2deb 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -48,6 +48,7 @@ #include <sys/mman.h> #include <sys/prctl.h> #include <sys/timerfd.h> +#include <sys/wait.h> #include <linux/bitops.h> #include <linux/hash.h> @@ -319,62 +320,6 @@ struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide) } #endif -int evlist__add_attrs(struct evlist *evlist, struct perf_event_attr *attrs, size_t nr_attrs) -{ - struct evsel *evsel, *n; - LIST_HEAD(head); - size_t i; - - for (i = 0; i < nr_attrs; i++) { - evsel = evsel__new_idx(attrs + i, evlist->core.nr_entries + i); - if (evsel == NULL) - goto out_delete_partial_list; - list_add_tail(&evsel->core.node, &head); - } - - evlist__splice_list_tail(evlist, &head); - - return 0; - -out_delete_partial_list: - __evlist__for_each_entry_safe(&head, n, evsel) - evsel__delete(evsel); - return -1; -} - -int __evlist__add_default_attrs(struct evlist *evlist, struct perf_event_attr *attrs, size_t nr_attrs) -{ - size_t i; - - for (i = 0; i < nr_attrs; i++) - event_attr_init(attrs + i); - - return evlist__add_attrs(evlist, attrs, nr_attrs); -} - -__weak int arch_evlist__add_default_attrs(struct evlist *evlist, - struct perf_event_attr *attrs, - size_t nr_attrs) -{ - if (!nr_attrs) - return 0; - - return __evlist__add_default_attrs(evlist, attrs, nr_attrs); -} - -struct evsel *evlist__find_tracepoint_by_id(struct evlist *evlist, int id) -{ - struct evsel *evsel; - - evlist__for_each_entry(evlist, evsel) { - if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT && - (int)evsel->core.attr.config == id) - return evsel; - } - - return NULL; -} - struct evsel *evlist__find_tracepoint_by_name(struct evlist *evlist, const char *name) { struct evsel *evsel; @@ -1199,11 +1144,6 @@ int evlist__set_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids) return ret; } -int evlist__set_tp_filter_pid(struct evlist *evlist, pid_t pid) -{ - return evlist__set_tp_filter_pids(evlist, 1, &pid); -} - int evlist__append_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids) { char *filter = asprintf__tp_filter_pids(npids, pids); @@ -1484,6 +1424,8 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const int child_ready_pipe[2], go_pipe[2]; char bf; + evlist->workload.cork_fd = -1; + if (pipe(child_ready_pipe) < 0) { perror("failed to create 'ready' pipe"); return -1; @@ -1536,7 +1478,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const * For cancelling the workload without actually running it, * the parent will just close workload.cork_fd, without writing * anything, i.e. read will return zero and we just exit() - * here. + * here (See evlist__cancel_workload()). */ if (ret != 1) { if (ret == -1) @@ -1600,7 +1542,7 @@ out_close_ready_pipe: int evlist__start_workload(struct evlist *evlist) { - if (evlist->workload.cork_fd > 0) { + if (evlist->workload.cork_fd >= 0) { char bf = 0; int ret; /* @@ -1611,12 +1553,24 @@ int evlist__start_workload(struct evlist *evlist) perror("unable to write to pipe"); close(evlist->workload.cork_fd); + evlist->workload.cork_fd = -1; return ret; } return 0; } +void evlist__cancel_workload(struct evlist *evlist) +{ + int status; + + if (evlist->workload.cork_fd >= 0) { + close(evlist->workload.cork_fd); + evlist->workload.cork_fd = -1; + waitpid(evlist->workload.pid, &status, WNOHANG); + } +} + int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample) { struct evsel *evsel = evlist__event2evsel(evlist, event); @@ -2619,7 +2573,8 @@ void evlist__uniquify_name(struct evlist *evlist) else attributes = empty_attributes; - if (asprintf(&new_name, "%s/%s/%s", pos->pmu_name, pos->name, attributes + 1)) { + if (asprintf(&new_name, "%s/%s/%s", pos->pmu ? pos->pmu->name : "", + pos->name, attributes + 1)) { free(pos->name); pos->name = new_name; } else { diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index bcc1c6984bb5..adddb1db1ad2 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -102,18 +102,6 @@ void evlist__delete(struct evlist *evlist); void evlist__add(struct evlist *evlist, struct evsel *entry); void evlist__remove(struct evlist *evlist, struct evsel *evsel); -int evlist__add_attrs(struct evlist *evlist, struct perf_event_attr *attrs, size_t nr_attrs); - -int __evlist__add_default_attrs(struct evlist *evlist, - struct perf_event_attr *attrs, size_t nr_attrs); - -int arch_evlist__add_default_attrs(struct evlist *evlist, - struct perf_event_attr *attrs, - size_t nr_attrs); - -#define evlist__add_default_attrs(evlist, array) \ - arch_evlist__add_default_attrs(evlist, array, ARRAY_SIZE(array)) - int arch_evlist__cmp(const struct evsel *lhs, const struct evsel *rhs); int evlist__add_dummy(struct evlist *evlist); @@ -144,7 +132,6 @@ int __evlist__set_tracepoints_handlers(struct evlist *evlist, __evlist__set_tracepoints_handlers(evlist, array, ARRAY_SIZE(array)) int evlist__set_tp_filter(struct evlist *evlist, const char *filter); -int evlist__set_tp_filter_pid(struct evlist *evlist, pid_t pid); int evlist__set_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids); int evlist__append_tp_filter(struct evlist *evlist, const char *filter); @@ -152,7 +139,6 @@ int evlist__append_tp_filter(struct evlist *evlist, const char *filter); int evlist__append_tp_filter_pid(struct evlist *evlist, pid_t pid); int evlist__append_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids); -struct evsel *evlist__find_tracepoint_by_id(struct evlist *evlist, int id); struct evsel *evlist__find_tracepoint_by_name(struct evlist *evlist, const char *name); int evlist__add_pollfd(struct evlist *evlist, int fd); @@ -186,6 +172,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const char *argv[], bool pipe_output, void (*exec_error)(int signo, siginfo_t *info, void *ucontext)); int evlist__start_workload(struct evlist *evlist); +void evlist__cancel_workload(struct evlist *evlist); struct option; diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index dbf9c8cee3c5..f745723d486b 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -5,12 +5,16 @@ * Parts came from builtin-{top,stat,record}.c, see those files for further * copyright notes. */ +/* + * Powerpc needs __SANE_USERSPACE_TYPES__ before <linux/types.h> to select + * 'int-ll64.h' and avoid compile warnings when printing __u64 with %llu. + */ +#define __SANE_USERSPACE_TYPES__ #include <byteswap.h> #include <errno.h> #include <inttypes.h> #include <linux/bitops.h> -#include <api/io.h> #include <api/fs/fs.h> #include <api/fs/tracing_path.h> #include <linux/hw_breakpoint.h> @@ -20,6 +24,7 @@ #include <linux/zalloc.h> #include <sys/ioctl.h> #include <sys/resource.h> +#include <sys/syscall.h> #include <sys/types.h> #include <dirent.h> #include <stdlib.h> @@ -51,6 +56,8 @@ #include "off_cpu.h" #include "pmu.h" #include "pmus.h" +#include "hwmon_pmu.h" +#include "tool_pmu.h" #include "rlimit.h" #include "../perf-sys.h" #include "util/parse-branch-options.h" @@ -64,46 +71,135 @@ #include <linux/ctype.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif struct perf_missing_features perf_missing_features; static clockid_t clockid; -static const char *const perf_tool_event__tool_names[PERF_TOOL_MAX] = { - NULL, - "duration_time", - "user_time", - "system_time", -}; - -const char *perf_tool_event__to_str(enum perf_tool_event ev) +static int evsel__no_extra_init(struct evsel *evsel __maybe_unused) { - if (ev > PERF_TOOL_NONE && ev < PERF_TOOL_MAX) - return perf_tool_event__tool_names[ev]; - - return NULL; + return 0; } -enum perf_tool_event perf_tool_event__from_str(const char *str) +static bool test_attr__enabled(void) { - int i; + static bool test_attr__enabled; + static bool test_attr__enabled_tested; + + if (!test_attr__enabled_tested) { + char *dir = getenv("PERF_TEST_ATTR"); - perf_tool_event__for_each_event(i) { - if (!strcmp(str, perf_tool_event__tool_names[i])) - return i; + test_attr__enabled = (dir != NULL); + test_attr__enabled_tested = true; } - return PERF_TOOL_NONE; + return test_attr__enabled; } +#define __WRITE_ASS(str, fmt, data) \ +do { \ + if (fprintf(file, #str "=%"fmt "\n", data) < 0) { \ + perror("test attr - failed to write event file"); \ + fclose(file); \ + return -1; \ + } \ +} while (0) -static int evsel__no_extra_init(struct evsel *evsel __maybe_unused) +#define WRITE_ASS(field, fmt) __WRITE_ASS(field, fmt, attr->field) + +static int store_event(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, + int fd, int group_fd, unsigned long flags) { + FILE *file; + char path[PATH_MAX]; + char *dir = getenv("PERF_TEST_ATTR"); + + snprintf(path, PATH_MAX, "%s/event-%d-%llu-%d", dir, + attr->type, attr->config, fd); + + file = fopen(path, "w+"); + if (!file) { + perror("test attr - failed to open event file"); + return -1; + } + + if (fprintf(file, "[event-%d-%llu-%d]\n", + attr->type, attr->config, fd) < 0) { + perror("test attr - failed to write event file"); + fclose(file); + return -1; + } + + /* syscall arguments */ + __WRITE_ASS(fd, "d", fd); + __WRITE_ASS(group_fd, "d", group_fd); + __WRITE_ASS(cpu, "d", cpu.cpu); + __WRITE_ASS(pid, "d", pid); + __WRITE_ASS(flags, "lu", flags); + + /* struct perf_event_attr */ + WRITE_ASS(type, PRIu32); + WRITE_ASS(size, PRIu32); + WRITE_ASS(config, "llu"); + WRITE_ASS(sample_period, "llu"); + WRITE_ASS(sample_type, "llu"); + WRITE_ASS(read_format, "llu"); + WRITE_ASS(disabled, "d"); + WRITE_ASS(inherit, "d"); + WRITE_ASS(pinned, "d"); + WRITE_ASS(exclusive, "d"); + WRITE_ASS(exclude_user, "d"); + WRITE_ASS(exclude_kernel, "d"); + WRITE_ASS(exclude_hv, "d"); + WRITE_ASS(exclude_idle, "d"); + WRITE_ASS(mmap, "d"); + WRITE_ASS(comm, "d"); + WRITE_ASS(freq, "d"); + WRITE_ASS(inherit_stat, "d"); + WRITE_ASS(enable_on_exec, "d"); + WRITE_ASS(task, "d"); + WRITE_ASS(watermark, "d"); + WRITE_ASS(precise_ip, "d"); + WRITE_ASS(mmap_data, "d"); + WRITE_ASS(sample_id_all, "d"); + WRITE_ASS(exclude_host, "d"); + WRITE_ASS(exclude_guest, "d"); + WRITE_ASS(exclude_callchain_kernel, "d"); + WRITE_ASS(exclude_callchain_user, "d"); + WRITE_ASS(mmap2, "d"); + WRITE_ASS(comm_exec, "d"); + WRITE_ASS(context_switch, "d"); + WRITE_ASS(write_backward, "d"); + WRITE_ASS(namespaces, "d"); + WRITE_ASS(use_clockid, "d"); + WRITE_ASS(wakeup_events, PRIu32); + WRITE_ASS(bp_type, PRIu32); + WRITE_ASS(config1, "llu"); + WRITE_ASS(config2, "llu"); + WRITE_ASS(branch_sample_type, "llu"); + WRITE_ASS(sample_regs_user, "llu"); + WRITE_ASS(sample_stack_user, PRIu32); + + fclose(file); return 0; } -void __weak test_attr__ready(void) { } +#undef __WRITE_ASS +#undef WRITE_ASS + +static void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, + int fd, int group_fd, unsigned long flags) +{ + int errno_saved = errno; + + if ((fd != -1) && store_event(attr, pid, cpu, fd, group_fd, flags)) { + pr_err("test attr FAILED"); + exit(128); + } + + errno = errno_saved; +} static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused) { @@ -296,9 +392,9 @@ void evsel__init(struct evsel *evsel, evsel->metric_events = NULL; evsel->per_pkg_mask = NULL; evsel->collect_stat = false; - evsel->pmu_name = NULL; evsel->group_pmu_name = NULL; evsel->skippable = false; + evsel->alternate_hw_config = PERF_COUNT_HW_MAX; } struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx) @@ -393,11 +489,6 @@ struct evsel *evsel__clone(struct evsel *orig) if (evsel->group_name == NULL) goto out_err; } - if (orig->pmu_name) { - evsel->pmu_name = strdup(orig->pmu_name); - if (evsel->pmu_name == NULL) - goto out_err; - } if (orig->group_pmu_name) { evsel->group_pmu_name = strdup(orig->group_pmu_name); if (evsel->group_pmu_name == NULL) @@ -421,7 +512,6 @@ struct evsel *evsel__clone(struct evsel *orig) evsel->core.leader = orig->core.leader; evsel->max_events = orig->max_events; - evsel->tool_event = orig->tool_event; free((char *)evsel->unit); evsel->unit = strdup(orig->unit); if (evsel->unit == NULL) @@ -445,6 +535,8 @@ struct evsel *evsel__clone(struct evsel *orig) if (evsel__copy_config_terms(evsel, orig) < 0) goto out_err; + evsel->alternate_hw_config = orig->alternate_hw_config; + return evsel; out_err: @@ -548,7 +640,6 @@ static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size) { int colon = 0, r = 0; struct perf_event_attr *attr = &evsel->core.attr; - bool exclude_guest_default = false; #define MOD_PRINT(context, mod) do { \ if (!attr->exclude_##context) { \ @@ -560,17 +651,15 @@ static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size) MOD_PRINT(kernel, 'k'); MOD_PRINT(user, 'u'); MOD_PRINT(hv, 'h'); - exclude_guest_default = true; } if (attr->precise_ip) { if (!colon) colon = ++r; r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp"); - exclude_guest_default = true; } - if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) { + if (attr->exclude_host || attr->exclude_guest) { MOD_PRINT(host, 'H'); MOD_PRINT(guest, 'G'); } @@ -617,11 +706,6 @@ static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size) return r + evsel__add_modifiers(evsel, bf + r, size - r); } -static int evsel__tool_name(enum perf_tool_event ev, char *bf, size_t size) -{ - return scnprintf(bf, size, "%s", perf_tool_event__to_str(ev)); -} - static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type) { int r; @@ -772,10 +856,7 @@ const char *evsel__name(struct evsel *evsel) break; case PERF_TYPE_SOFTWARE: - if (evsel__is_tool(evsel)) - evsel__tool_name(evsel__tool_event(evsel), bf, sizeof(bf)); - else - evsel__sw_name(evsel, bf, sizeof(bf)); + evsel__sw_name(evsel, bf, sizeof(bf)); break; case PERF_TYPE_TRACEPOINT: @@ -786,6 +867,10 @@ const char *evsel__name(struct evsel *evsel) evsel__bp_name(evsel, bf, sizeof(bf)); break; + case PERF_PMU_TYPE_TOOL: + scnprintf(bf, sizeof(bf), "%s", evsel__tool_pmu_event_name(evsel)); + break; + default: scnprintf(bf, sizeof(bf), "unknown attr type: %d", evsel->core.attr.type); @@ -811,7 +896,7 @@ const char *evsel__metric_id(const struct evsel *evsel) return evsel->metric_id; if (evsel__is_tool(evsel)) - return perf_tool_event__to_str(evsel__tool_event(evsel)); + return evsel__tool_pmu_event_name(evsel); return "unknown"; } @@ -862,7 +947,6 @@ static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *o { bool function = evsel__is_function_event(evsel); struct perf_event_attr *attr = &evsel->core.attr; - const char *arch = perf_env__arch(evsel__env(evsel)); evsel__set_sample_bit(evsel, CALLCHAIN); @@ -893,6 +977,8 @@ static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *o if (param->record_mode == CALLCHAIN_DWARF) { if (!function) { + const char *arch = perf_env__arch(evsel__env(evsel)); + evsel__set_sample_bit(evsel, REGS_USER); evsel__set_sample_bit(evsel, STACK_USER); if (opts->sample_user_regs && @@ -1150,7 +1236,7 @@ void evsel__config(struct evsel *evsel, struct record_opts *opts, bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread; attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1; - attr->inherit = !opts->no_inherit; + attr->inherit = target__has_cpu(&opts->target) ? 0 : !opts->no_inherit; attr->write_backward = opts->overwrite ? 1 : 0; attr->read_format = PERF_FORMAT_LOST; @@ -1172,7 +1258,15 @@ void evsel__config(struct evsel *evsel, struct record_opts *opts, */ if (leader->core.nr_members > 1) { attr->read_format |= PERF_FORMAT_GROUP; - attr->inherit = 0; + } + + /* + * Inherit + SAMPLE_READ requires SAMPLE_TID in the read_format + */ + if (attr->inherit) { + evsel__set_sample_bit(evsel, TID); + evsel->core.attr.read_format |= + PERF_FORMAT_ID; } } @@ -1494,7 +1588,6 @@ void evsel__exit(struct evsel *evsel) zfree(&evsel->group_name); zfree(&evsel->name); zfree(&evsel->filter); - zfree(&evsel->pmu_name); zfree(&evsel->group_pmu_name); zfree(&evsel->unit); zfree(&evsel->metric_id); @@ -1503,8 +1596,8 @@ void evsel__exit(struct evsel *evsel) evsel->per_pkg_mask = NULL; zfree(&evsel->metric_events); perf_evsel__object.fini(evsel); - if (evsel__tool_event(evsel) == PERF_TOOL_SYSTEM_TIME || - evsel__tool_event(evsel) == PERF_TOOL_USER_TIME) + if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || + evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) xyarray__delete(evsel->start_times); } @@ -1684,171 +1777,31 @@ static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread) return evsel__process_group_data(leader, cpu_map_idx, thread, data); } -static bool read_until_char(struct io *io, char e) -{ - int c; - - do { - c = io__get_char(io); - if (c == -1) - return false; - } while (c != e); - return true; -} - -static int read_stat_field(int fd, struct perf_cpu cpu, int field, __u64 *val) -{ - char buf[256]; - struct io io; - int i; - - io__init(&io, fd, buf, sizeof(buf)); - - /* Skip lines to relevant CPU. */ - for (i = -1; i < cpu.cpu; i++) { - if (!read_until_char(&io, '\n')) - return -EINVAL; - } - /* Skip to "cpu". */ - if (io__get_char(&io) != 'c') return -EINVAL; - if (io__get_char(&io) != 'p') return -EINVAL; - if (io__get_char(&io) != 'u') return -EINVAL; - - /* Skip N of cpuN. */ - if (!read_until_char(&io, ' ')) - return -EINVAL; - - i = 1; - while (true) { - if (io__get_dec(&io, val) != ' ') - break; - if (field == i) - return 0; - i++; - } - return -EINVAL; -} - -static int read_pid_stat_field(int fd, int field, __u64 *val) -{ - char buf[256]; - struct io io; - int c, i; - - io__init(&io, fd, buf, sizeof(buf)); - if (io__get_dec(&io, val) != ' ') - return -EINVAL; - if (field == 1) - return 0; - - /* Skip comm. */ - if (io__get_char(&io) != '(' || !read_until_char(&io, ')')) - return -EINVAL; - if (field == 2) - return -EINVAL; /* String can't be returned. */ - - /* Skip state */ - if (io__get_char(&io) != ' ' || io__get_char(&io) == -1) - return -EINVAL; - if (field == 3) - return -EINVAL; /* String can't be returned. */ - - /* Loop over numeric fields*/ - if (io__get_char(&io) != ' ') - return -EINVAL; - - i = 4; - while (true) { - c = io__get_dec(&io, val); - if (c == -1) - return -EINVAL; - if (c == -2) { - /* Assume a -ve was read */ - c = io__get_dec(&io, val); - *val *= -1; - } - if (c != ' ') - return -EINVAL; - if (field == i) - return 0; - i++; - } - return -EINVAL; -} - -static int evsel__read_tool(struct evsel *evsel, int cpu_map_idx, int thread) +bool __evsel__match(const struct evsel *evsel, u32 type, u64 config) { - __u64 *start_time, cur_time, delta_start; - int fd, err = 0; - struct perf_counts_values *count; - bool adjust = false; - count = perf_counts(evsel->counts, cpu_map_idx, thread); + u32 e_type = evsel->core.attr.type; + u64 e_config = evsel->core.attr.config; - switch (evsel__tool_event(evsel)) { - case PERF_TOOL_DURATION_TIME: - /* - * Pretend duration_time is only on the first CPU and thread, or - * else aggregation will scale duration_time by the number of - * CPUs/threads. - */ - start_time = &evsel->start_time; - if (cpu_map_idx == 0 && thread == 0) - cur_time = rdclock(); - else - cur_time = *start_time; - break; - case PERF_TOOL_USER_TIME: - case PERF_TOOL_SYSTEM_TIME: { - bool system = evsel__tool_event(evsel) == PERF_TOOL_SYSTEM_TIME; - - start_time = xyarray__entry(evsel->start_times, cpu_map_idx, thread); - fd = FD(evsel, cpu_map_idx, thread); - lseek(fd, SEEK_SET, 0); - if (evsel->pid_stat) { - /* The event exists solely on 1 CPU. */ - if (cpu_map_idx == 0) - err = read_pid_stat_field(fd, system ? 15 : 14, &cur_time); - else - cur_time = 0; - } else { - /* The event is for all threads. */ - if (thread == 0) { - struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, - cpu_map_idx); - - err = read_stat_field(fd, cpu, system ? 3 : 1, &cur_time); - } else { - cur_time = 0; - } - } - adjust = true; - break; - } - case PERF_TOOL_NONE: - case PERF_TOOL_MAX: - default: - err = -EINVAL; + if (e_type != type) { + return type == PERF_TYPE_HARDWARE && evsel->pmu && evsel->pmu->is_core && + evsel->alternate_hw_config == config; } - if (err) - return err; - delta_start = cur_time - *start_time; - if (adjust) { - __u64 ticks_per_sec = sysconf(_SC_CLK_TCK); + if ((type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) && + perf_pmus__supports_extended_type()) + e_config &= PERF_HW_EVENT_MASK; - delta_start *= 1000000000 / ticks_per_sec; - } - count->val = delta_start; - count->ena = count->run = delta_start; - count->lost = 0; - return 0; + return e_config == config; } int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread) { if (evsel__is_tool(evsel)) - return evsel__read_tool(evsel, cpu_map_idx, thread); + return evsel__tool_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_hwmon(evsel)) + return evsel__hwmon_pmu_read(evsel, cpu_map_idx, thread); if (evsel__is_retire_lat(evsel)) return evsel__read_retire_lat(evsel, cpu_map_idx, thread); @@ -2042,6 +1995,7 @@ static struct perf_thread_map *empty_thread_map; static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, struct perf_thread_map *threads) { + int ret = 0; int nthreads = perf_thread_map__nr(threads); if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) || @@ -2072,23 +2026,21 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0) return -ENOMEM; - if ((evsel__tool_event(evsel) == PERF_TOOL_SYSTEM_TIME || - evsel__tool_event(evsel) == PERF_TOOL_USER_TIME) && - !evsel->start_times) { - evsel->start_times = xyarray__new(perf_cpu_map__nr(cpus), nthreads, sizeof(__u64)); - if (!evsel->start_times) - return -ENOMEM; - } + if (evsel__is_tool(evsel)) + ret = evsel__tool_pmu_prepare_open(evsel, cpus, nthreads); evsel->open_flags = PERF_FLAG_FD_CLOEXEC; if (evsel->cgrp) evsel->open_flags |= PERF_FLAG_PID_CGROUP; - return 0; + return ret; } static void evsel__disable_missing_features(struct evsel *evsel) { + if (perf_missing_features.inherit_sample_read && evsel->core.attr.inherit && + (evsel->core.attr.sample_type & PERF_SAMPLE_READ)) + evsel->core.attr.inherit = 0; if (perf_missing_features.branch_counters) evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_COUNTERS; if (perf_missing_features.read_lost) @@ -2138,120 +2090,346 @@ int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, return err; } -bool evsel__detect_missing_features(struct evsel *evsel) +static bool has_attr_feature(struct perf_event_attr *attr, unsigned long flags) +{ + int fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, /*cpu=*/-1, + /*group_fd=*/-1, flags); + close(fd); + + if (fd < 0) { + attr->exclude_kernel = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, /*cpu=*/-1, + /*group_fd=*/-1, flags); + close(fd); + } + + if (fd < 0) { + attr->exclude_hv = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, /*cpu=*/-1, + /*group_fd=*/-1, flags); + close(fd); + } + + if (fd < 0) { + attr->exclude_guest = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, /*cpu=*/-1, + /*group_fd=*/-1, flags); + close(fd); + } + + attr->exclude_kernel = 0; + attr->exclude_guest = 0; + attr->exclude_hv = 0; + + return fd >= 0; +} + +static void evsel__detect_missing_pmu_features(struct evsel *evsel) { + struct perf_event_attr attr = { + .type = evsel->core.attr.type, + .config = evsel->core.attr.config, + .disabled = 1, + }; + struct perf_pmu *pmu = evsel->pmu; + int old_errno; + + old_errno = errno; + + if (pmu == NULL) + pmu = evsel->pmu = evsel__find_pmu(evsel); + + if (pmu == NULL || pmu->missing_features.checked) + goto out; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are kernel core limitation but + * specific to PMUs with branch stack. So we can detect with the given + * hardware event and stop on the first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.exclude_guest = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + pmu->missing_features.exclude_guest = true; + pr_debug2("switching off exclude_guest for PMU %s\n", pmu->name); + +found: + pmu->missing_features.checked = true; +out: + errno = old_errno; +} + +static void evsel__detect_missing_brstack_features(struct evsel *evsel) +{ + static bool detection_done = false; + struct perf_event_attr attr = { + .type = evsel->core.attr.type, + .config = evsel->core.attr.config, + .disabled = 1, + .sample_type = PERF_SAMPLE_BRANCH_STACK, + .sample_period = 1000, + }; + int old_errno; + + if (detection_done) + return; + + old_errno = errno; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are PMU specific limitation + * so we can detect with the given hardware event and stop on the + * first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_COUNTERS; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.branch_counters = true; + pr_debug2("switching off branch counters support\n"); + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_HW_INDEX; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.branch_hw_idx = true; + pr_debug2("switching off branch HW index support\n"); + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_NO_CYCLES | PERF_SAMPLE_BRANCH_NO_FLAGS; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.lbr_flags = true; + pr_debug2_peo("switching off branch sample type no (cycles/flags)\n"); + +found: + detection_done = true; + errno = old_errno; +} + +static bool evsel__detect_missing_features(struct evsel *evsel) +{ + static bool detection_done = false; + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_TASK_CLOCK, + .disabled = 1, + }; + int old_errno; + + evsel__detect_missing_pmu_features(evsel); + + if (evsel__has_br_stack(evsel)) + evsel__detect_missing_brstack_features(evsel); + + if (detection_done) + goto check; + + old_errno = errno; + /* * Must probe features in the order they were added to the - * perf_event_attr interface. + * perf_event_attr interface. These are kernel core limitation + * not PMU-specific so we can detect with a software event and + * stop on the first one succeeded. */ - if (!perf_missing_features.branch_counters && - (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS)) { - perf_missing_features.branch_counters = true; - pr_debug2("switching off branch counters support\n"); + + /* Please add new feature detection here. */ + + attr.inherit = true; + attr.sample_type = PERF_SAMPLE_READ; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.inherit_sample_read = true; + pr_debug2("Using PERF_SAMPLE_READ / :S modifier is not compatible with inherit, falling back to no-inherit.\n"); + attr.inherit = false; + attr.sample_type = 0; + + attr.read_format = PERF_FORMAT_LOST; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.read_lost = true; + pr_debug2("switching off PERF_FORMAT_LOST support\n"); + attr.read_format = 0; + + attr.sample_type = PERF_SAMPLE_WEIGHT_STRUCT; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.weight_struct = true; + pr_debug2("switching off weight struct support\n"); + attr.sample_type = 0; + + attr.sample_type = PERF_SAMPLE_CODE_PAGE_SIZE; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.code_page_size = true; + pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support\n"); + attr.sample_type = 0; + + attr.sample_type = PERF_SAMPLE_DATA_PAGE_SIZE; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.data_page_size = true; + pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support\n"); + attr.sample_type = 0; + + attr.cgroup = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.cgroup = true; + pr_debug2_peo("Kernel has no cgroup sampling support\n"); + attr.cgroup = 0; + + attr.aux_output = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.aux_output = true; + pr_debug2_peo("Kernel has no attr.aux_output support\n"); + attr.aux_output = 0; + + attr.bpf_event = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.bpf = true; + pr_debug2_peo("switching off bpf_event\n"); + attr.bpf_event = 0; + + attr.ksymbol = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.ksymbol = true; + pr_debug2_peo("switching off ksymbol\n"); + attr.ksymbol = 0; + + attr.write_backward = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.write_backward = true; + pr_debug2_peo("switching off write_backward\n"); + attr.write_backward = 0; + + attr.use_clockid = 1; + attr.clockid = CLOCK_MONOTONIC; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.clockid = true; + pr_debug2_peo("switching off clockid\n"); + attr.use_clockid = 0; + attr.clockid = 0; + + if (has_attr_feature(&attr, /*flags=*/PERF_FLAG_FD_CLOEXEC)) + goto found; + perf_missing_features.cloexec = true; + pr_debug2_peo("switching off cloexec flag\n"); + + attr.mmap2 = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.mmap2 = true; + pr_debug2_peo("switching off mmap2\n"); + attr.mmap2 = 0; + + /* set this unconditionally? */ + perf_missing_features.sample_id_all = true; + pr_debug2_peo("switching off sample_id_all\n"); + + attr.inherit = 1; + attr.read_format = PERF_FORMAT_GROUP; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.group_read = true; + pr_debug2_peo("switching off group read\n"); + attr.inherit = 0; + attr.read_format = 0; + +found: + detection_done = true; + errno = old_errno; + +check: + if (evsel->core.attr.inherit && + (evsel->core.attr.sample_type & PERF_SAMPLE_READ) && + perf_missing_features.inherit_sample_read) return true; - } else if (!perf_missing_features.read_lost && - (evsel->core.attr.read_format & PERF_FORMAT_LOST)) { - perf_missing_features.read_lost = true; - pr_debug2("switching off PERF_FORMAT_LOST support\n"); + + if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + perf_missing_features.branch_counters) return true; - } else if (!perf_missing_features.weight_struct && - (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) { - perf_missing_features.weight_struct = true; - pr_debug2("switching off weight struct support\n"); + + if ((evsel->core.attr.read_format & PERF_FORMAT_LOST) && + perf_missing_features.read_lost) return true; - } else if (!perf_missing_features.code_page_size && - (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) { - perf_missing_features.code_page_size = true; - pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n"); - return false; - } else if (!perf_missing_features.data_page_size && - (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) { - perf_missing_features.data_page_size = true; - pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n"); - return false; - } else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) { - perf_missing_features.cgroup = true; - pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n"); - return false; - } else if (!perf_missing_features.branch_hw_idx && - (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) { - perf_missing_features.branch_hw_idx = true; - pr_debug2("switching off branch HW index support\n"); + + if ((evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT) && + perf_missing_features.weight_struct) return true; - } else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) { - perf_missing_features.aux_output = true; - pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n"); - return false; - } else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) { - perf_missing_features.bpf = true; - pr_debug2_peo("switching off bpf_event\n"); + + if (evsel->core.attr.use_clockid && evsel->core.attr.clockid != CLOCK_MONOTONIC && + !perf_missing_features.clockid) { + perf_missing_features.clockid_wrong = true; return true; - } else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) { - perf_missing_features.ksymbol = true; - pr_debug2_peo("switching off ksymbol\n"); + } + + if (evsel->core.attr.use_clockid && perf_missing_features.clockid) return true; - } else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) { - perf_missing_features.write_backward = true; - pr_debug2_peo("switching off write_backward\n"); - return false; - } else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) { - perf_missing_features.clockid_wrong = true; - pr_debug2_peo("switching off clockid\n"); + + if ((evsel->open_flags & PERF_FLAG_FD_CLOEXEC) && + perf_missing_features.cloexec) return true; - } else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) { - perf_missing_features.clockid = true; - pr_debug2_peo("switching off use_clockid\n"); + + if (evsel->core.attr.mmap2 && perf_missing_features.mmap2) return true; - } else if (!perf_missing_features.cloexec && (evsel->open_flags & PERF_FLAG_FD_CLOEXEC)) { - perf_missing_features.cloexec = true; - pr_debug2_peo("switching off cloexec flag\n"); + + if ((evsel->core.attr.branch_sample_type & (PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_NO_CYCLES)) && + perf_missing_features.lbr_flags) return true; - } else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) { - perf_missing_features.mmap2 = true; - pr_debug2_peo("switching off mmap2\n"); + + if (evsel->core.attr.inherit && (evsel->core.attr.read_format & PERF_FORMAT_GROUP) && + perf_missing_features.group_read) return true; - } else if (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host) { - if (evsel->pmu == NULL) - evsel->pmu = evsel__find_pmu(evsel); - - if (evsel->pmu) - evsel->pmu->missing_features.exclude_guest = true; - else { - /* we cannot find PMU, disable attrs now */ - evsel->core.attr.exclude_host = false; - evsel->core.attr.exclude_guest = false; - } - if (evsel->exclude_GH) { - pr_debug2_peo("PMU has no exclude_host/guest support, bailing out\n"); - return false; - } - if (!perf_missing_features.exclude_guest) { - perf_missing_features.exclude_guest = true; - pr_debug2_peo("switching off exclude_guest, exclude_host\n"); - } + if (evsel->core.attr.ksymbol && perf_missing_features.ksymbol) return true; - } else if (!perf_missing_features.sample_id_all) { - perf_missing_features.sample_id_all = true; - pr_debug2_peo("switching off sample_id_all\n"); + + if (evsel->core.attr.bpf_event && perf_missing_features.bpf) return true; - } else if (!perf_missing_features.lbr_flags && - (evsel->core.attr.branch_sample_type & - (PERF_SAMPLE_BRANCH_NO_CYCLES | - PERF_SAMPLE_BRANCH_NO_FLAGS))) { - perf_missing_features.lbr_flags = true; - pr_debug2_peo("switching off branch sample type no (cycles/flags)\n"); + + if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) && + perf_missing_features.branch_hw_idx) return true; - } else if (!perf_missing_features.group_read && - evsel->core.attr.inherit && - (evsel->core.attr.read_format & PERF_FORMAT_GROUP) && - evsel__is_group_leader(evsel)) { - perf_missing_features.group_read = true; - pr_debug2_peo("switching off group read\n"); + + if (evsel->core.attr.sample_id_all && perf_missing_features.sample_id_all) + return true; + + return false; +} + +static bool evsel__handle_error_quirks(struct evsel *evsel, int error) +{ + /* + * AMD core PMU tries to forward events with precise_ip to IBS PMU + * implicitly. But IBS PMU has more restrictions so it can fail with + * supported event attributes. Let's forward it back to the core PMU + * by clearing precise_ip only if it's from precise_max (:P). + */ + if ((error == -EINVAL || error == -ENOENT) && x86__is_amd_cpu() && + evsel->core.attr.precise_ip && evsel->precise_max) { + evsel->core.attr.precise_ip = 0; + pr_debug2_peo("removing precise_ip on AMD\n"); + display_attr(&evsel->core.attr); return true; - } else { - return false; } + + return false; } static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, @@ -2262,13 +2440,6 @@ static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int pid = -1, err, old_errno; enum rlimit_action set_rlimit = NO_CHANGE; - if (evsel__tool_event(evsel) == PERF_TOOL_DURATION_TIME) { - if (evsel->core.attr.sample_period) /* no sampling */ - return -EINVAL; - evsel->start_time = rdclock(); - return 0; - } - if (evsel__is_retire_lat(evsel)) return tpebs_start(evsel->evlist); @@ -2293,6 +2464,17 @@ fallback_missing_features: pr_debug3("Opening: %s\n", evsel__name(evsel)); display_attr(&evsel->core.attr); + if (evsel__is_tool(evsel)) { + return evsel__tool_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + } + if (evsel__is_hwmon(evsel)) { + return evsel__hwmon_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + } + for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) { for (thread = 0; thread < nthreads; thread++) { @@ -2304,46 +2486,6 @@ retry_open: if (!evsel->cgrp && !evsel->core.system_wide) pid = perf_thread_map__pid(threads, thread); - if (evsel__tool_event(evsel) == PERF_TOOL_USER_TIME || - evsel__tool_event(evsel) == PERF_TOOL_SYSTEM_TIME) { - bool system = evsel__tool_event(evsel) == PERF_TOOL_SYSTEM_TIME; - __u64 *start_time = NULL; - - if (evsel->core.attr.sample_period) { - /* no sampling */ - err = -EINVAL; - goto out_close; - } - if (pid > -1) { - char buf[64]; - - snprintf(buf, sizeof(buf), "/proc/%d/stat", pid); - fd = open(buf, O_RDONLY); - evsel->pid_stat = true; - } else { - fd = open("/proc/stat", O_RDONLY); - } - FD(evsel, idx, thread) = fd; - if (fd < 0) { - err = -errno; - goto out_close; - } - start_time = xyarray__entry(evsel->start_times, idx, thread); - if (pid > -1) { - err = read_pid_stat_field(fd, system ? 15 : 14, - start_time); - } else { - struct perf_cpu cpu; - - cpu = perf_cpu_map__cpu(evsel->core.cpus, idx); - err = read_stat_field(fd, cpu, system ? 3 : 1, - start_time); - } - if (err) - goto out_close; - continue; - } - group_fd = get_group_fd(evsel, idx, thread); if (group_fd == -2) { @@ -2352,8 +2494,6 @@ retry_open: goto out_close; } - test_attr__ready(); - /* Debug message used by test scripts */ pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx", pid, perf_cpu_map__cpu(cpus, idx).cpu, group_fd, evsel->open_flags); @@ -2374,7 +2514,7 @@ retry_open: bpf_counter__install_pe(evsel, idx, fd); - if (unlikely(test_attr__enabled)) { + if (unlikely(test_attr__enabled())) { test_attr__open(&evsel->core.attr, pid, perf_cpu_map__cpu(cpus, idx), fd, group_fd, evsel->open_flags); @@ -2415,9 +2555,6 @@ retry_open: return 0; try_fallback: - if (evsel__precise_ip_fallback(evsel)) - goto retry_open; - if (evsel__ignore_missing_thread(evsel, perf_cpu_map__nr(cpus), idx, threads, thread, err)) { /* We just removed 1 thread, so lower the upper nthreads limit. */ @@ -2434,11 +2571,15 @@ try_fallback: if (err == -EMFILE && rlimit__increase_nofile(&set_rlimit)) goto retry_open; - if (err != -EINVAL || idx > 0 || thread > 0) - goto out_close; + if (err == -EOPNOTSUPP && evsel__precise_ip_fallback(evsel)) + goto retry_open; - if (evsel__detect_missing_features(evsel)) + if (err == -EINVAL && evsel__detect_missing_features(evsel)) goto fallback_missing_features; + + if (evsel__handle_error_quirks(evsel, err)) + goto retry_open; + out_close: if (err) threads->err_thread = thread; @@ -3245,6 +3386,27 @@ bool evsel__fallback(struct evsel *evsel, struct target *target, int err, evsel->core.attr.exclude_hv = 1; return true; + } else if (err == EOPNOTSUPP && !evsel->core.attr.exclude_guest && + !evsel->exclude_GH) { + const char *name = evsel__name(evsel); + char *new_name; + const char *sep = ":"; + + /* Is there already the separator in the name. */ + if (strchr(name, '/') || + (strchr(name, ':') && !evsel->is_libpfm_event)) + sep = ""; + + if (asprintf(&new_name, "%s%sH", name, sep) < 0) + return false; + + free(evsel->name); + evsel->name = new_name; + /* Apple M1 requires exclude_guest */ + scnprintf(msg, msgsize, "trying to fall back to excluding guest samples"); + evsel->core.attr.exclude_guest = 1; + + return true; } return false; @@ -3415,7 +3577,7 @@ int evsel__open_strerror(struct evsel *evsel, struct target *target, return scnprintf(msg, size, "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n" - "/bin/dmesg | grep -i perf may provide additional information.\n", + "\"dmesg | grep -i perf\" may provide additional information.\n", err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel)); } diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 15e745a9a798..04934a7af174 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -11,6 +11,7 @@ #include <perf/evsel.h> #include "symbol_conf.h" #include "pmus.h" +#include "pmu.h" struct bpf_object; struct cgroup; @@ -22,25 +23,9 @@ struct target; struct hashmap; struct bperf_leader_bpf; struct bperf_follower_bpf; -struct perf_pmu; typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); -enum perf_tool_event { - PERF_TOOL_NONE = 0, - PERF_TOOL_DURATION_TIME = 1, - PERF_TOOL_USER_TIME = 2, - PERF_TOOL_SYSTEM_TIME = 3, - - PERF_TOOL_MAX, -}; - -const char *perf_tool_event__to_str(enum perf_tool_event ev); -enum perf_tool_event perf_tool_event__from_str(const char *str); - -#define perf_tool_event__for_each_event(ev) \ - for ((ev) = PERF_TOOL_DURATION_TIME; (ev) < PERF_TOOL_MAX; ev++) - /** struct evsel - event selector * * @evlist - evlist this evsel is in, if it is in one. @@ -72,7 +57,6 @@ struct evsel { struct { char *name; char *group_name; - const char *pmu_name; const char *group_pmu_name; #ifdef HAVE_LIBTRACEEVENT struct tep_event *tp_format; @@ -83,7 +67,6 @@ struct evsel { const char *unit; struct cgroup *cgrp; const char *metric_id; - enum perf_tool_event tool_event; /* parse modifier helper */ int exclude_GH; int sample_read; @@ -102,6 +85,7 @@ struct evsel { int bpf_fd; struct bpf_object *bpf_obj; struct list_head config_terms; + u64 alternate_hw_config; }; /* @@ -183,7 +167,7 @@ struct evsel { unsigned long open_flags; int precise_ip_original; - /* for missing_features */ + /* The PMU the event is from. Used for missing_features, PMU name, etc. */ struct perf_pmu *pmu; /* For tool events */ @@ -221,6 +205,7 @@ struct perf_missing_features { bool weight_struct; bool read_lost; bool branch_counters; + bool inherit_sample_read; }; extern struct perf_missing_features perf_missing_features; @@ -320,21 +305,11 @@ const char *evsel__name(struct evsel *evsel); bool evsel__name_is(struct evsel *evsel, const char *name); const char *evsel__metric_id(const struct evsel *evsel); -static inline bool evsel__is_tool(const struct evsel *evsel) -{ - return evsel->tool_event != PERF_TOOL_NONE; -} - static inline bool evsel__is_retire_lat(const struct evsel *evsel) { return evsel->retire_lat; } -static inline enum perf_tool_event evsel__tool_event(const struct evsel *evsel) -{ - return evsel->tool_event; -} - const char *evsel__group_name(struct evsel *evsel); int evsel__group_desc(struct evsel *evsel, char *buf, size_t size); @@ -368,7 +343,6 @@ int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus, void evsel__close(struct evsel *evsel); int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, struct perf_thread_map *threads); -bool evsel__detect_missing_features(struct evsel *evsel); bool evsel__precise_ip_fallback(struct evsel *evsel); @@ -393,26 +367,10 @@ u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sam struct tep_format_field *evsel__field(struct evsel *evsel, const char *name); struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name); -static inline bool __evsel__match(const struct evsel *evsel, u32 type, u64 config) -{ - if (evsel->core.attr.type != type) - return false; - - if ((type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) && - perf_pmus__supports_extended_type()) - return (evsel->core.attr.config & PERF_HW_EVENT_MASK) == config; - - return evsel->core.attr.config == config; -} +bool __evsel__match(const struct evsel *evsel, u32 type, u64 config); #define evsel__match(evsel, t, c) __evsel__match(evsel, PERF_TYPE_##t, PERF_COUNT_##c) -static inline bool evsel__match2(struct evsel *e1, struct evsel *e2) -{ - return (e1->core.attr.type == e2->core.attr.type) && - (e1->core.attr.config == e2->core.attr.config); -} - int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread); int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale); diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c index c2c0500d5da9..86b7f46f9e2a 100644 --- a/tools/perf/util/evsel_fprintf.c +++ b/tools/perf/util/evsel_fprintf.c @@ -14,7 +14,7 @@ #include "dso.h" #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif static int comma_fprintf(FILE *fp, bool *first, const char *fmt, ...) diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index b2536a59c44e..f289044a1f7c 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -5,25 +5,22 @@ #include <stdlib.h> #include <string.h> #include "metricgroup.h" -#include "cpumap.h" -#include "cputopo.h" #include "debug.h" #include "evlist.h" #include "expr.h" +#include "smt.h" +#include "tool_pmu.h" #include <util/expr-bison.h> #include <util/expr-flex.h> #include "util/hashmap.h" #include "util/header.h" #include "util/pmu.h" -#include "smt.h" -#include "tsc.h" -#include <api/fs/fs.h> +#include <perf/cpumap.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/zalloc.h> #include <ctype.h> #include <math.h> -#include "pmu.h" struct expr_id_data { union { @@ -393,90 +390,26 @@ double expr_id_data__source_count(const struct expr_id_data *data) return data->val.source_count; } -#if !defined(__i386__) && !defined(__x86_64__) -double arch_get_tsc_freq(void) -{ - return 0.0; -} -#endif - -static double has_pmem(void) -{ - static bool has_pmem, cached; - const char *sysfs = sysfs__mountpoint(); - char path[PATH_MAX]; - - if (!cached) { - snprintf(path, sizeof(path), "%s/firmware/acpi/tables/NFIT", sysfs); - has_pmem = access(path, F_OK) == 0; - cached = true; - } - return has_pmem ? 1.0 : 0.0; -} - double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx) { - const struct cpu_topology *topology; double result = NAN; + enum tool_pmu_event ev = tool_pmu__str_to_event(literal + 1); - if (!strcmp("#num_cpus", literal)) { - result = cpu__max_present_cpu().cpu; - goto out; - } - if (!strcmp("#num_cpus_online", literal)) { - struct perf_cpu_map *online = cpu_map__online(); - - if (online) - result = perf_cpu_map__nr(online); - goto out; - } + if (ev != TOOL_PMU__EVENT_NONE) { + u64 count; - if (!strcasecmp("#system_tsc_freq", literal)) { - result = arch_get_tsc_freq(); - goto out; - } + if (tool_pmu__read_event(ev, &count)) + result = count; + else + pr_err("Failure to read '%s'", literal); - /* - * Assume that topology strings are consistent, such as CPUs "0-1" - * wouldn't be listed as "0,1", and so after deduplication the number of - * these strings gives an indication of the number of packages, dies, - * etc. - */ - if (!strcasecmp("#smt_on", literal)) { - result = smt_on() ? 1.0 : 0.0; - goto out; - } - if (!strcmp("#core_wide", literal)) { + } else if (!strcmp("#core_wide", literal)) { result = core_wide(ctx->system_wide, ctx->user_requested_cpu_list) ? 1.0 : 0.0; - goto out; - } - if (!strcmp("#num_packages", literal)) { - topology = online_topology(); - result = topology->package_cpus_lists; - goto out; - } - if (!strcmp("#num_dies", literal)) { - topology = online_topology(); - result = topology->die_cpus_lists; - goto out; - } - if (!strcmp("#num_cores", literal)) { - topology = online_topology(); - result = topology->core_cpus_lists; - goto out; - } - if (!strcmp("#slots", literal)) { - result = perf_pmu__cpu_slots_per_cycle(); - goto out; - } - if (!strcmp("#has_pmem", literal)) { - result = has_pmem(); - goto out; + } else { + pr_err("Unrecognized literal '%s'", literal); } - pr_err("Unrecognized literal '%s'", literal); -out: pr_debug2("literal: %s = %f\n", literal, result); return result; } @@ -523,8 +456,8 @@ double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx __maybe_unused, bool compute_ids __maybe_unused, const char *test_id) { double ret; - struct perf_pmu *pmu = perf_pmus__find_core_pmu(); - char *cpuid = perf_pmu__getcpuid(pmu); + struct perf_cpu cpu = {-1}; + char *cpuid = get_cpuid_allow_env_override(cpu); if (!cpuid) return NAN; diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c index c8f6bee1fa61..cdce7f173d00 100644 --- a/tools/perf/util/genelf.c +++ b/tools/perf/util/genelf.c @@ -16,7 +16,7 @@ #include <inttypes.h> #include <fcntl.h> #include <err.h> -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT #include <dwarf.h> #endif @@ -499,7 +499,7 @@ jit_write_elf(int fd, uint64_t load_addr, const char *sym, shdr->sh_size = sizeof(bnote); shdr->sh_entsize = 0; -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT if (debug && nr_debug_entries) { retval = jit_add_debug_info(e, load_addr, debug, nr_debug_entries); if (retval) diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h index 4e2e4f40e134..9f0b875d6548 100644 --- a/tools/perf/util/genelf.h +++ b/tools/perf/util/genelf.h @@ -8,7 +8,7 @@ int jit_write_elf(int fd, uint64_t code_addr, const char *sym, const void *code, int csize, void *debug, int nr_debug_entries, void *unwinding, uint64_t unwinding_header_size, uint64_t unwinding_size); -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT /* genelf_debug.c */ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_entries); #endif diff --git a/tools/perf/util/hashmap.h b/tools/perf/util/hashmap.h index c12f8320e668..0c4f155e8eb7 100644 --- a/tools/perf/util/hashmap.h +++ b/tools/perf/util/hashmap.h @@ -166,8 +166,8 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); * @bkt: integer used as a bucket loop cursor */ #define hashmap__for_each_entry(map, cur, bkt) \ - for (bkt = 0; bkt < map->cap; bkt++) \ - for (cur = map->buckets[bkt]; cur; cur = cur->next) + for (bkt = 0; bkt < (map)->cap; bkt++) \ + for (cur = (map)->buckets[bkt]; cur; cur = cur->next) /* * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe @@ -178,8 +178,8 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); * @bkt: integer used as a bucket loop cursor */ #define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \ - for (bkt = 0; bkt < map->cap; bkt++) \ - for (cur = map->buckets[bkt]; \ + for (bkt = 0; bkt < (map)->cap; bkt++) \ + for (cur = (map)->buckets[bkt]; \ cur && ({tmp = cur->next; true; }); \ cur = tmp) @@ -190,19 +190,19 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); * @key: key to iterate entries for */ #define hashmap__for_each_key_entry(map, cur, _key) \ - for (cur = map->buckets \ - ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \ + for (cur = (map)->buckets \ + ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \ : NULL; \ cur; \ cur = cur->next) \ - if (map->equal_fn(cur->key, (_key), map->ctx)) + if ((map)->equal_fn(cur->key, (_key), (map)->ctx)) #define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \ - for (cur = map->buckets \ - ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \ + for (cur = (map)->buckets \ + ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \ : NULL; \ cur && ({ tmp = cur->next; true; }); \ cur = tmp) \ - if (map->equal_fn(cur->key, (_key), map->ctx)) + if ((map)->equal_fn(cur->key, (_key), (map)->ctx)) #endif /* __LIBBPF_HASHMAP_H */ diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index a6386d12afd7..3451e542b69a 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -58,7 +58,7 @@ #include <internal/lib.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif /* @@ -819,11 +819,31 @@ static int write_group_desc(struct feat_fd *ff, * Each architecture should provide a more precise id string that * can be use to match the architecture's "mapfile". */ -char * __weak get_cpuid_str(struct perf_pmu *pmu __maybe_unused) +char * __weak get_cpuid_str(struct perf_cpu cpu __maybe_unused) { return NULL; } +char *get_cpuid_allow_env_override(struct perf_cpu cpu) +{ + char *cpuid; + static bool printed; + + cpuid = getenv("PERF_CPUID"); + if (cpuid) + cpuid = strdup(cpuid); + if (!cpuid) + cpuid = get_cpuid_str(cpu); + if (!cpuid) + return NULL; + + if (!printed) { + pr_debug("Using CPUID %s\n", cpuid); + printed = true; + } + return cpuid; +} + /* Return zero when the cpuid from the mapfile.csv matches the * cpuid string generated on this platform. * Otherwise return non-zero. @@ -856,18 +876,19 @@ int __weak strcmp_cpuid_str(const char *mapcpuid, const char *cpuid) * default get_cpuid(): nothing gets recorded * actual implementation must be in arch/$(SRCARCH)/util/header.c */ -int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused) +int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused, + struct perf_cpu cpu __maybe_unused) { return ENOSYS; /* Not implemented */ } -static int write_cpuid(struct feat_fd *ff, - struct evlist *evlist __maybe_unused) +static int write_cpuid(struct feat_fd *ff, struct evlist *evlist) { + struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); char buffer[64]; int ret; - ret = get_cpuid(buffer, sizeof(buffer)); + ret = get_cpuid(buffer, sizeof(buffer), cpu); if (ret) return -1; @@ -987,57 +1008,6 @@ static int write_dir_format(struct feat_fd *ff, return do_write(ff, &data->dir.version, sizeof(data->dir.version)); } -/* - * Check whether a CPU is online - * - * Returns: - * 1 -> if CPU is online - * 0 -> if CPU is offline - * -1 -> error case - */ -int is_cpu_online(unsigned int cpu) -{ - char *str; - size_t strlen; - char buf[256]; - int status = -1; - struct stat statbuf; - - snprintf(buf, sizeof(buf), - "/sys/devices/system/cpu/cpu%d", cpu); - if (stat(buf, &statbuf) != 0) - return 0; - - /* - * Check if /sys/devices/system/cpu/cpux/online file - * exists. Some cases cpu0 won't have online file since - * it is not expected to be turned off generally. - * In kernels without CONFIG_HOTPLUG_CPU, this - * file won't exist - */ - snprintf(buf, sizeof(buf), - "/sys/devices/system/cpu/cpu%d/online", cpu); - if (stat(buf, &statbuf) != 0) - return 1; - - /* - * Read online file using sysfs__read_str. - * If read or open fails, return -1. - * If read succeeds, return value from file - * which gets stored in "str" - */ - snprintf(buf, sizeof(buf), - "devices/system/cpu/cpu%d/online", cpu); - - if (sysfs__read_str(buf, &str, &strlen) < 0) - return status; - - status = atoi(str); - - free(str); - return status; -} - #ifdef HAVE_LIBBPF_SUPPORT static int write_bpf_prog_info(struct feat_fd *ff, struct evlist *evlist __maybe_unused) diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h index a63a361f20f4..5201af6305f4 100644 --- a/tools/perf/util/header.h +++ b/tools/perf/util/header.h @@ -10,7 +10,13 @@ #include <linux/bitmap.h> #include <linux/types.h> #include "env.h" -#include "pmu.h" +#include <perf/cpumap.h> + +struct evlist; +union perf_event; +struct perf_header; +struct perf_session; +struct perf_tool; enum { HEADER_RESERVED = 0, /* always cleared */ @@ -91,8 +97,6 @@ struct perf_pipe_file_header { u64 size; }; -struct perf_header; - int perf_file_header__read(struct perf_file_header *header, struct perf_header *ph, int fd); @@ -124,11 +128,6 @@ struct perf_header_feature_ops { bool synthesize; }; -struct evlist; -struct perf_session; -struct perf_tool; -union perf_event; - extern const char perf_version_string[]; int perf_session__read_header(struct perf_session *session); @@ -196,14 +195,16 @@ int write_padded(struct feat_fd *fd, const void *bf, #define MAX_CACHE_LVL 4 -int is_cpu_online(unsigned int cpu); int build_caches_for_cpu(u32 cpu, struct cpu_cache_level caches[], u32 *cntp); /* * arch specific callback */ -int get_cpuid(char *buffer, size_t sz); +int get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu); + +char *get_cpuid_str(struct perf_cpu cpu); + +char *get_cpuid_allow_env_override(struct perf_cpu cpu); -char *get_cpuid_str(struct perf_pmu *pmu __maybe_unused); int strcmp_cpuid_str(const char *s1, const char *s2); #endif /* __PERF_HEADER_H */ diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index f387e85a0087..fff134565801 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -218,6 +218,9 @@ void hists__calc_col_len(struct hists *hists, struct hist_entry *h) hists__new_col_len(hists, HISTC_LOCAL_P_STAGE_CYC, 13); hists__new_col_len(hists, HISTC_GLOBAL_P_STAGE_CYC, 13); hists__new_col_len(hists, HISTC_ADDR, BITS_PER_LONG / 4 + 2); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_PREDICTED, 9); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_ABORT, 5); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_CYCLES, 6); if (symbol_conf.nanosecs) hists__new_col_len(hists, HISTC_TIME, 16); diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index 7d7ae94b4b31..1131056924d9 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -87,6 +87,9 @@ enum hist_column { HISTC_TYPE_OFFSET, HISTC_SYMBOL_OFFSET, HISTC_TYPE_CACHELINE, + HISTC_CALLCHAIN_BRANCH_PREDICTED, + HISTC_CALLCHAIN_BRANCH_ABORT, + HISTC_CALLCHAIN_BRANCH_CYCLES, HISTC_NR_COLS, /* Last entry */ }; diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c new file mode 100644 index 000000000000..e61429b38ba7 --- /dev/null +++ b/tools/perf/util/hwmon_pmu.c @@ -0,0 +1,832 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "hashmap.h" +#include "hwmon_pmu.h" +#include "pmu.h" +#include <internal/xyarray.h> +#include <internal/threadmap.h> +#include <perf/threadmap.h> +#include <sys/types.h> +#include <assert.h> +#include <ctype.h> +#include <dirent.h> +#include <fcntl.h> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <api/fs/fs.h> +#include <api/io.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/zalloc.h> + +/** Strings that correspond to enum hwmon_type. */ +static const char * const hwmon_type_strs[HWMON_TYPE_MAX] = { + NULL, + "cpu", + "curr", + "energy", + "fan", + "humidity", + "in", + "intrusion", + "power", + "pwm", + "temp", +}; +#define LONGEST_HWMON_TYPE_STR "intrusion" + +/** Strings that correspond to enum hwmon_item. */ +static const char * const hwmon_item_strs[HWMON_ITEM__MAX] = { + NULL, + "accuracy", + "alarm", + "auto_channels_temp", + "average", + "average_highest", + "average_interval", + "average_interval_max", + "average_interval_min", + "average_lowest", + "average_max", + "average_min", + "beep", + "cap", + "cap_hyst", + "cap_max", + "cap_min", + "crit", + "crit_hyst", + "div", + "emergency", + "emergency_hist", + "enable", + "fault", + "freq", + "highest", + "input", + "label", + "lcrit", + "lcrit_hyst", + "lowest", + "max", + "max_hyst", + "min", + "min_hyst", + "mod", + "offset", + "pulses", + "rated_max", + "rated_min", + "reset_history", + "target", + "type", + "vid", +}; +#define LONGEST_HWMON_ITEM_STR "average_interval_max" + +static const char *const hwmon_units[HWMON_TYPE_MAX] = { + NULL, + "V", /* cpu */ + "A", /* curr */ + "J", /* energy */ + "rpm", /* fan */ + "%", /* humidity */ + "V", /* in */ + "", /* intrusion */ + "W", /* power */ + "Hz", /* pwm */ + "'C", /* temp */ +}; + +struct hwmon_pmu { + struct perf_pmu pmu; + struct hashmap events; + int hwmon_dir_fd; +}; + +/** + * union hwmon_pmu_event_key: Key for hwmon_pmu->events as such each key + * represents an event. + * + * Related hwmon files start <type><number> that this key represents. + */ +union hwmon_pmu_event_key { + long type_and_num; + struct { + int num :16; + enum hwmon_type type :8; + }; +}; + +/** + * struct hwmon_pmu_event_value: Value in hwmon_pmu->events. + * + * Hwmon files are of the form <type><number>_<item> and may have a suffix + * _alarm. + */ +struct hwmon_pmu_event_value { + /** @items: which item files are present. */ + DECLARE_BITMAP(items, HWMON_ITEM__MAX); + /** @alarm_items: which item files are present. */ + DECLARE_BITMAP(alarm_items, HWMON_ITEM__MAX); + /** @label: contents of <type><number>_label if present. */ + char *label; + /** @name: name computed from label of the form <type>_<label>. */ + char *name; +}; + +bool perf_pmu__is_hwmon(const struct perf_pmu *pmu) +{ + return pmu && pmu->type >= PERF_PMU_TYPE_HWMON_START && + pmu->type <= PERF_PMU_TYPE_HWMON_END; +} + +bool evsel__is_hwmon(const struct evsel *evsel) +{ + return perf_pmu__is_hwmon(evsel->pmu); +} + +static size_t hwmon_pmu__event_hashmap_hash(long key, void *ctx __maybe_unused) +{ + return ((union hwmon_pmu_event_key)key).type_and_num; +} + +static bool hwmon_pmu__event_hashmap_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return ((union hwmon_pmu_event_key)key1).type_and_num == + ((union hwmon_pmu_event_key)key2).type_and_num; +} + +static int hwmon_strcmp(const void *a, const void *b) +{ + const char *sa = a; + const char * const *sb = b; + + return strcmp(sa, *sb); +} + +bool parse_hwmon_filename(const char *filename, + enum hwmon_type *type, + int *number, + enum hwmon_item *item, + bool *alarm) +{ + char fn_type[24]; + const char **elem; + const char *fn_item = NULL; + size_t fn_item_len; + + assert(strlen(LONGEST_HWMON_TYPE_STR) < sizeof(fn_type)); + strlcpy(fn_type, filename, sizeof(fn_type)); + for (size_t i = 0; fn_type[i] != '\0'; i++) { + if (fn_type[i] >= '0' && fn_type[i] <= '9') { + fn_type[i] = '\0'; + *number = strtoul(&filename[i], (char **)&fn_item, 10); + if (*fn_item == '_') + fn_item++; + break; + } + if (fn_type[i] == '_') { + fn_type[i] = '\0'; + *number = -1; + fn_item = &filename[i + 1]; + break; + } + } + if (fn_item == NULL || fn_type[0] == '\0' || (item != NULL && fn_item[0] == '\0')) { + pr_debug3("hwmon_pmu: not a hwmon file '%s'\n", filename); + return false; + } + elem = bsearch(&fn_type, hwmon_type_strs + 1, ARRAY_SIZE(hwmon_type_strs) - 1, + sizeof(hwmon_type_strs[0]), hwmon_strcmp); + if (!elem) { + pr_debug3("hwmon_pmu: not a hwmon type '%s' in file name '%s'\n", + fn_type, filename); + return false; + } + + *type = elem - &hwmon_type_strs[0]; + if (!item) + return true; + + *alarm = false; + fn_item_len = strlen(fn_item); + if (fn_item_len > 6 && !strcmp(&fn_item[fn_item_len - 6], "_alarm")) { + assert(strlen(LONGEST_HWMON_ITEM_STR) < sizeof(fn_type)); + strlcpy(fn_type, fn_item, fn_item_len - 5); + fn_item = fn_type; + *alarm = true; + } + elem = bsearch(fn_item, hwmon_item_strs + 1, ARRAY_SIZE(hwmon_item_strs) - 1, + sizeof(hwmon_item_strs[0]), hwmon_strcmp); + if (!elem) { + pr_debug3("hwmon_pmu: not a hwmon item '%s' in file name '%s'\n", + fn_item, filename); + return false; + } + *item = elem - &hwmon_item_strs[0]; + return true; +} + +static void fix_name(char *p) +{ + char *s = strchr(p, '\n'); + + if (s) + *s = '\0'; + + while (*p != '\0') { + if (strchr(" :,/\n\t", *p)) + *p = '_'; + else + *p = tolower(*p); + p++; + } +} + +static int hwmon_pmu__read_events(struct hwmon_pmu *pmu) +{ + DIR *dir; + struct dirent *ent; + int dup_fd, err = 0; + struct hashmap_entry *cur, *tmp; + size_t bkt; + + if (pmu->pmu.sysfs_aliases_loaded) + return 0; + + /* Use a dup-ed fd as closedir will close it. */ + dup_fd = dup(pmu->hwmon_dir_fd); + if (dup_fd == -1) + return -ENOMEM; + + dir = fdopendir(dup_fd); + if (!dir) { + close(dup_fd); + return -ENOMEM; + } + + while ((ent = readdir(dir)) != NULL) { + enum hwmon_type type; + int number; + enum hwmon_item item; + bool alarm; + union hwmon_pmu_event_key key = { .type_and_num = 0 }; + struct hwmon_pmu_event_value *value; + + if (ent->d_type != DT_REG) + continue; + + if (!parse_hwmon_filename(ent->d_name, &type, &number, &item, &alarm)) { + pr_debug3("Not a hwmon file '%s'\n", ent->d_name); + continue; + } + key.num = number; + key.type = type; + if (!hashmap__find(&pmu->events, key.type_and_num, &value)) { + value = zalloc(sizeof(*value)); + if (!value) { + err = -ENOMEM; + goto err_out; + } + err = hashmap__add(&pmu->events, key.type_and_num, value); + if (err) { + free(value); + err = -ENOMEM; + goto err_out; + } + } + __set_bit(item, alarm ? value->alarm_items : value->items); + if (item == HWMON_ITEM_LABEL) { + char buf[128]; + int fd = openat(pmu->hwmon_dir_fd, ent->d_name, O_RDONLY); + ssize_t read_len; + + if (fd < 0) + continue; + + read_len = read(fd, buf, sizeof(buf)); + + while (read_len > 0 && buf[read_len - 1] == '\n') + read_len--; + + if (read_len > 0) + buf[read_len] = '\0'; + + if (buf[0] == '\0') { + pr_debug("hwmon_pmu: empty label file %s %s\n", + pmu->pmu.name, ent->d_name); + close(fd); + continue; + } + value->label = strdup(buf); + if (!value->label) { + pr_debug("hwmon_pmu: memory allocation failure\n"); + close(fd); + continue; + } + snprintf(buf, sizeof(buf), "%s_%s", hwmon_type_strs[type], value->label); + fix_name(buf); + value->name = strdup(buf); + if (!value->name) + pr_debug("hwmon_pmu: memory allocation failure\n"); + close(fd); + } + } + hashmap__for_each_entry_safe((&pmu->events), cur, tmp, bkt) { + union hwmon_pmu_event_key key = { + .type_and_num = cur->key, + }; + struct hwmon_pmu_event_value *value = cur->pvalue; + + if (!test_bit(HWMON_ITEM_INPUT, value->items)) { + pr_debug("hwmon_pmu: removing event '%s%d' that has no input file\n", + hwmon_type_strs[key.type], key.num); + hashmap__delete(&pmu->events, key.type_and_num, &key, &value); + zfree(&value->label); + zfree(&value->name); + free(value); + } + } + pmu->pmu.sysfs_aliases_loaded = true; + +err_out: + closedir(dir); + return err; +} + +struct perf_pmu *hwmon_pmu__new(struct list_head *pmus, int hwmon_dir, const char *sysfs_name, const char *name) +{ + char buf[32]; + struct hwmon_pmu *hwm; + + hwm = zalloc(sizeof(*hwm)); + if (!hwm) + return NULL; + + hwm->hwmon_dir_fd = hwmon_dir; + hwm->pmu.type = PERF_PMU_TYPE_HWMON_START + strtoul(sysfs_name + 5, NULL, 10); + if (hwm->pmu.type > PERF_PMU_TYPE_HWMON_END) { + pr_err("Unable to encode hwmon type from %s in valid PMU type\n", sysfs_name); + goto err_out; + } + snprintf(buf, sizeof(buf), "hwmon_%s", name); + fix_name(buf + 6); + hwm->pmu.name = strdup(buf); + if (!hwm->pmu.name) + goto err_out; + hwm->pmu.alias_name = strdup(sysfs_name); + if (!hwm->pmu.alias_name) + goto err_out; + hwm->pmu.cpus = perf_cpu_map__new("0"); + if (!hwm->pmu.cpus) + goto err_out; + INIT_LIST_HEAD(&hwm->pmu.format); + INIT_LIST_HEAD(&hwm->pmu.aliases); + INIT_LIST_HEAD(&hwm->pmu.caps); + hashmap__init(&hwm->events, hwmon_pmu__event_hashmap_hash, + hwmon_pmu__event_hashmap_equal, /*ctx=*/NULL); + + list_add_tail(&hwm->pmu.list, pmus); + return &hwm->pmu; +err_out: + free((char *)hwm->pmu.name); + free(hwm->pmu.alias_name); + free(hwm); + close(hwmon_dir); + return NULL; +} + +void hwmon_pmu__exit(struct perf_pmu *pmu) +{ + struct hwmon_pmu *hwm = container_of(pmu, struct hwmon_pmu, pmu); + struct hashmap_entry *cur, *tmp; + size_t bkt; + + hashmap__for_each_entry_safe((&hwm->events), cur, tmp, bkt) { + struct hwmon_pmu_event_value *value = cur->pvalue; + + zfree(&value->label); + zfree(&value->name); + free(value); + } + hashmap__clear(&hwm->events); + close(hwm->hwmon_dir_fd); +} + +static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, size_t out_buf_len, + union hwmon_pmu_event_key key, + const unsigned long *items, bool is_alarm) +{ + size_t bit; + char buf[64]; + size_t len = 0; + + for_each_set_bit(bit, items, HWMON_ITEM__MAX) { + int fd; + + if (bit == HWMON_ITEM_LABEL || bit == HWMON_ITEM_INPUT) + continue; + + snprintf(buf, sizeof(buf), "%s%d_%s%s", + hwmon_type_strs[key.type], + key.num, + hwmon_item_strs[bit], + is_alarm ? "_alarm" : ""); + fd = openat(hwm->hwmon_dir_fd, buf, O_RDONLY); + if (fd > 0) { + ssize_t read_len = read(fd, buf, sizeof(buf)); + + while (read_len > 0 && buf[read_len - 1] == '\n') + read_len--; + + if (read_len > 0) { + long long val; + + buf[read_len] = '\0'; + val = strtoll(buf, /*endptr=*/NULL, 10); + len += snprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", + len == 0 ? " " : ", ", + hwmon_item_strs[bit], + is_alarm ? "_alarm" : "", + (double)val / 1000.0, + hwmon_units[key.type]); + } + close(fd); + } + } + return len; +} + +int hwmon_pmu__for_each_event(struct perf_pmu *pmu, void *state, pmu_event_callback cb) +{ + struct hwmon_pmu *hwm = container_of(pmu, struct hwmon_pmu, pmu); + struct hashmap_entry *cur; + size_t bkt; + + if (hwmon_pmu__read_events(hwm)) + return false; + + hashmap__for_each_entry((&hwm->events), cur, bkt) { + static const char *const hwmon_scale_units[HWMON_TYPE_MAX] = { + NULL, + "0.001V", /* cpu */ + "0.001A", /* curr */ + "0.001J", /* energy */ + "1rpm", /* fan */ + "0.001%", /* humidity */ + "0.001V", /* in */ + NULL, /* intrusion */ + "0.001W", /* power */ + "1Hz", /* pwm */ + "0.001'C", /* temp */ + }; + static const char *const hwmon_desc[HWMON_TYPE_MAX] = { + NULL, + "CPU core reference voltage", /* cpu */ + "Current", /* curr */ + "Cumulative energy use", /* energy */ + "Fan", /* fan */ + "Humidity", /* humidity */ + "Voltage", /* in */ + "Chassis intrusion detection", /* intrusion */ + "Power use", /* power */ + "Pulse width modulation fan control", /* pwm */ + "Temperature", /* temp */ + }; + char alias_buf[64]; + char desc_buf[256]; + char encoding_buf[128]; + union hwmon_pmu_event_key key = { + .type_and_num = cur->key, + }; + struct hwmon_pmu_event_value *value = cur->pvalue; + struct pmu_event_info info = { + .pmu = pmu, + .name = value->name, + .alias = alias_buf, + .scale_unit = hwmon_scale_units[key.type], + .desc = desc_buf, + .long_desc = NULL, + .encoding_desc = encoding_buf, + .topic = "hwmon", + .pmu_name = pmu->name, + .event_type_desc = "Hwmon event", + }; + int ret; + size_t len; + + len = snprintf(alias_buf, sizeof(alias_buf), "%s%d", + hwmon_type_strs[key.type], key.num); + if (!info.name) { + info.name = info.alias; + info.alias = NULL; + } + + len = snprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", + hwmon_desc[key.type], + pmu->name + 6, + value->label ?: info.name); + + len += hwmon_pmu__describe_items(hwm, desc_buf + len, sizeof(desc_buf) - len, + key, value->items, /*is_alarm=*/false); + + len += hwmon_pmu__describe_items(hwm, desc_buf + len, sizeof(desc_buf) - len, + key, value->alarm_items, /*is_alarm=*/true); + + snprintf(encoding_buf, sizeof(encoding_buf), "%s/config=0x%lx/", + pmu->name, cur->key); + + ret = cb(state, &info); + if (ret) + return ret; + } + return 0; +} + +size_t hwmon_pmu__num_events(struct perf_pmu *pmu) +{ + struct hwmon_pmu *hwm = container_of(pmu, struct hwmon_pmu, pmu); + + hwmon_pmu__read_events(hwm); + return hashmap__size(&hwm->events); +} + +bool hwmon_pmu__have_event(struct perf_pmu *pmu, const char *name) +{ + struct hwmon_pmu *hwm = container_of(pmu, struct hwmon_pmu, pmu); + enum hwmon_type type; + int number; + union hwmon_pmu_event_key key = { .type_and_num = 0 }; + struct hashmap_entry *cur; + size_t bkt; + + if (!parse_hwmon_filename(name, &type, &number, /*item=*/NULL, /*is_alarm=*/NULL)) + return false; + + if (hwmon_pmu__read_events(hwm)) + return false; + + key.type = type; + key.num = number; + if (hashmap_find(&hwm->events, key.type_and_num, /*value=*/NULL)) + return true; + if (key.num != -1) + return false; + /* Item is of form <type>_ which means we should match <type>_<label>. */ + hashmap__for_each_entry((&hwm->events), cur, bkt) { + struct hwmon_pmu_event_value *value = cur->pvalue; + + key.type_and_num = cur->key; + if (key.type == type && value->name && !strcasecmp(name, value->name)) + return true; + } + return false; +} + +static int hwmon_pmu__config_term(const struct hwmon_pmu *hwm, + struct perf_event_attr *attr, + struct parse_events_term *term, + struct parse_events_error *err) +{ + if (term->type_term == PARSE_EVENTS__TERM_TYPE_USER) { + enum hwmon_type type; + int number; + + if (parse_hwmon_filename(term->config, &type, &number, + /*item=*/NULL, /*is_alarm=*/NULL)) { + if (number == -1) { + /* + * Item is of form <type>_ which means we should + * match <type>_<label>. + */ + struct hashmap_entry *cur; + size_t bkt; + + attr->config = 0; + hashmap__for_each_entry((&hwm->events), cur, bkt) { + union hwmon_pmu_event_key key = { + .type_and_num = cur->key, + }; + struct hwmon_pmu_event_value *value = cur->pvalue; + + if (key.type == type && value->name && + !strcasecmp(term->config, value->name)) { + attr->config = key.type_and_num; + break; + } + } + if (attr->config == 0) + return -EINVAL; + } else { + union hwmon_pmu_event_key key = { + .type_and_num = 0, + }; + + key.type = type; + key.num = number; + attr->config = key.type_and_num; + } + return 0; + } + } + if (err) { + char *err_str; + + parse_events_error__handle(err, term->err_val, + asprintf(&err_str, + "unexpected hwmon event term (%s) %s", + parse_events__term_type_str(term->type_term), + term->config) < 0 + ? strdup("unexpected hwmon event term") + : err_str, + NULL); + } + return -EINVAL; +} + +int hwmon_pmu__config_terms(const struct perf_pmu *pmu, + struct perf_event_attr *attr, + struct parse_events_terms *terms, + struct parse_events_error *err) +{ + struct hwmon_pmu *hwm = container_of(pmu, struct hwmon_pmu, pmu); + struct parse_events_term *term; + int ret; + + ret = hwmon_pmu__read_events(hwm); + if (ret) + return ret; + + list_for_each_entry(term, &terms->terms, list) { + if (hwmon_pmu__config_term(hwm, attr, term, err)) + return -EINVAL; + } + + return 0; + +} + +int hwmon_pmu__check_alias(struct parse_events_terms *terms, struct perf_pmu_info *info, + struct parse_events_error *err) +{ + struct parse_events_term *term = + list_first_entry(&terms->terms, struct parse_events_term, list); + + if (term->type_term == PARSE_EVENTS__TERM_TYPE_USER) { + enum hwmon_type type; + int number; + + if (parse_hwmon_filename(term->config, &type, &number, + /*item=*/NULL, /*is_alarm=*/NULL)) { + info->unit = hwmon_units[type]; + if (type == HWMON_TYPE_FAN || type == HWMON_TYPE_PWM || + type == HWMON_TYPE_INTRUSION) + info->scale = 1; + else + info->scale = 0.001; + } + return 0; + } + if (err) { + char *err_str; + + parse_events_error__handle(err, term->err_val, + asprintf(&err_str, + "unexpected hwmon event term (%s) %s", + parse_events__term_type_str(term->type_term), + term->config) < 0 + ? strdup("unexpected hwmon event term") + : err_str, + NULL); + } + return -EINVAL; +} + +int perf_pmus__read_hwmon_pmus(struct list_head *pmus) +{ + char *line = NULL; + DIR *class_hwmon_dir; + struct dirent *class_hwmon_ent; + char buf[PATH_MAX]; + const char *sysfs = sysfs__mountpoint(); + + if (!sysfs) + return 0; + + scnprintf(buf, sizeof(buf), "%s/class/hwmon/", sysfs); + class_hwmon_dir = opendir(buf); + if (!class_hwmon_dir) + return 0; + + while ((class_hwmon_ent = readdir(class_hwmon_dir)) != NULL) { + size_t line_len; + int hwmon_dir, name_fd; + struct io io; + + if (class_hwmon_ent->d_type != DT_LNK) + continue; + + scnprintf(buf, sizeof(buf), "%s/class/hwmon/%s", sysfs, class_hwmon_ent->d_name); + hwmon_dir = open(buf, O_DIRECTORY); + if (hwmon_dir == -1) { + pr_debug("hwmon_pmu: not a directory: '%s/class/hwmon/%s'\n", + sysfs, class_hwmon_ent->d_name); + continue; + } + name_fd = openat(hwmon_dir, "name", O_RDONLY); + if (name_fd == -1) { + pr_debug("hwmon_pmu: failure to open '%s/class/hwmon/%s/name'\n", + sysfs, class_hwmon_ent->d_name); + close(hwmon_dir); + continue; + } + io__init(&io, name_fd, buf, sizeof(buf)); + io__getline(&io, &line, &line_len); + if (line_len > 0 && line[line_len - 1] == '\n') + line[line_len - 1] = '\0'; + hwmon_pmu__new(pmus, hwmon_dir, class_hwmon_ent->d_name, line); + close(name_fd); + } + free(line); + closedir(class_hwmon_dir); + return 0; +} + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) + +int evsel__hwmon_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx) +{ + struct hwmon_pmu *hwm = container_of(evsel->pmu, struct hwmon_pmu, pmu); + union hwmon_pmu_event_key key = { + .type_and_num = evsel->core.attr.config, + }; + int idx = 0, thread = 0, nthreads, err = 0; + + nthreads = perf_thread_map__nr(threads); + for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) { + for (thread = 0; thread < nthreads; thread++) { + char buf[64]; + int fd; + + snprintf(buf, sizeof(buf), "%s%d_input", + hwmon_type_strs[key.type], key.num); + + fd = openat(hwm->hwmon_dir_fd, buf, O_RDONLY); + FD(evsel, idx, thread) = fd; + if (fd < 0) { + err = -errno; + goto out_close; + } + } + } + return 0; +out_close: + if (err) + threads->err_thread = thread; + + do { + while (--thread >= 0) { + if (FD(evsel, idx, thread) >= 0) + close(FD(evsel, idx, thread)); + FD(evsel, idx, thread) = -1; + } + thread = nthreads; + } while (--idx >= 0); + return err; +} + +int evsel__hwmon_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) +{ + char buf[32]; + int fd; + ssize_t len; + struct perf_counts_values *count, *old_count = NULL; + + if (evsel->prev_raw_counts) + old_count = perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); + + count = perf_counts(evsel->counts, cpu_map_idx, thread); + fd = FD(evsel, cpu_map_idx, thread); + len = pread(fd, buf, sizeof(buf), 0); + if (len <= 0) { + count->lost++; + return -EINVAL; + } + buf[len] = '\0'; + if (old_count) { + count->val = old_count->val + strtoll(buf, NULL, 10); + count->run = old_count->run + 1; + count->ena = old_count->ena + 1; + } else { + count->val = strtoll(buf, NULL, 10); + count->run++; + count->ena++; + } + return 0; +} diff --git a/tools/perf/util/hwmon_pmu.h b/tools/perf/util/hwmon_pmu.h new file mode 100644 index 000000000000..882566846df4 --- /dev/null +++ b/tools/perf/util/hwmon_pmu.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __HWMON_PMU_H +#define __HWMON_PMU_H + +#include "pmu.h" +#include <stdbool.h> + +struct list_head; +struct perf_thread_map; + +/** + * enum hwmon_type: + * + * As described in Documentation/hwmon/sysfs-interface.rst hwmon events are + * defined over multiple files of the form <type><num>_<item>. This enum + * captures potential <type> values. + * + * This enum is exposed for testing. + */ +enum hwmon_type { + HWMON_TYPE_NONE, + + HWMON_TYPE_CPU, + HWMON_TYPE_CURR, + HWMON_TYPE_ENERGY, + HWMON_TYPE_FAN, + HWMON_TYPE_HUMIDITY, + HWMON_TYPE_IN, + HWMON_TYPE_INTRUSION, + HWMON_TYPE_POWER, + HWMON_TYPE_PWM, + HWMON_TYPE_TEMP, + + HWMON_TYPE_MAX +}; + +/** + * enum hwmon_item: + * + * Similar to enum hwmon_type but describes the item part of a a sysfs filename. + * + * This enum is exposed for testing. + */ +enum hwmon_item { + HWMON_ITEM_NONE, + + HWMON_ITEM_ACCURACY, + HWMON_ITEM_ALARM, + HWMON_ITEM_AUTO_CHANNELS_TEMP, + HWMON_ITEM_AVERAGE, + HWMON_ITEM_AVERAGE_HIGHEST, + HWMON_ITEM_AVERAGE_INTERVAL, + HWMON_ITEM_AVERAGE_INTERVAL_MAX, + HWMON_ITEM_AVERAGE_INTERVAL_MIN, + HWMON_ITEM_AVERAGE_LOWEST, + HWMON_ITEM_AVERAGE_MAX, + HWMON_ITEM_AVERAGE_MIN, + HWMON_ITEM_BEEP, + HWMON_ITEM_CAP, + HWMON_ITEM_CAP_HYST, + HWMON_ITEM_CAP_MAX, + HWMON_ITEM_CAP_MIN, + HWMON_ITEM_CRIT, + HWMON_ITEM_CRIT_HYST, + HWMON_ITEM_DIV, + HWMON_ITEM_EMERGENCY, + HWMON_ITEM_EMERGENCY_HIST, + HWMON_ITEM_ENABLE, + HWMON_ITEM_FAULT, + HWMON_ITEM_FREQ, + HWMON_ITEM_HIGHEST, + HWMON_ITEM_INPUT, + HWMON_ITEM_LABEL, + HWMON_ITEM_LCRIT, + HWMON_ITEM_LCRIT_HYST, + HWMON_ITEM_LOWEST, + HWMON_ITEM_MAX, + HWMON_ITEM_MAX_HYST, + HWMON_ITEM_MIN, + HWMON_ITEM_MIN_HYST, + HWMON_ITEM_MOD, + HWMON_ITEM_OFFSET, + HWMON_ITEM_PULSES, + HWMON_ITEM_RATED_MAX, + HWMON_ITEM_RATED_MIN, + HWMON_ITEM_RESET_HISTORY, + HWMON_ITEM_TARGET, + HWMON_ITEM_TYPE, + HWMON_ITEM_VID, + + HWMON_ITEM__MAX, +}; + +bool perf_pmu__is_hwmon(const struct perf_pmu *pmu); +bool evsel__is_hwmon(const struct evsel *evsel); + +/** + * parse_hwmon_filename() - Parse filename into constituent parts. + * + * @filename: To be parsed, of the form <type><number>_<item>. + * @type: The type defined from the parsed file name. + * @number: The number of the type, for example there may be more than 1 fan. + * @item: A hwmon <type><number> may have multiple associated items. + * @alarm: Is the filename for an alarm value? + * + * An example of a hwmon filename is "temp1_input". The type is temp for a + * temperature value. The number is 1. The item within the file is an input + * value - the temperature itself. This file doesn't contain an alarm value. + * + * Exposed for testing. + */ +bool parse_hwmon_filename(const char *filename, + enum hwmon_type *type, + int *number, + enum hwmon_item *item, + bool *alarm); + +/** + * hwmon_pmu__new() - Allocate and construct a hwmon PMU. + * + * @pmus: The list of PMUs to be added to. + * @hwmon_dir: An O_DIRECTORY file descriptor for a hwmon directory. + * @sysfs_name: Name of the hwmon sysfs directory like hwmon0. + * @name: The contents of the "name" file in the hwmon directory. + * + * Exposed for testing. Regular construction should happen via + * perf_pmus__read_hwmon_pmus. + */ +struct perf_pmu *hwmon_pmu__new(struct list_head *pmus, int hwmon_dir, + const char *sysfs_name, const char *name); +void hwmon_pmu__exit(struct perf_pmu *pmu); + +int hwmon_pmu__for_each_event(struct perf_pmu *pmu, void *state, pmu_event_callback cb); +size_t hwmon_pmu__num_events(struct perf_pmu *pmu); +bool hwmon_pmu__have_event(struct perf_pmu *pmu, const char *name); +int hwmon_pmu__config_terms(const struct perf_pmu *pmu, + struct perf_event_attr *attr, + struct parse_events_terms *terms, + struct parse_events_error *err); +int hwmon_pmu__check_alias(struct parse_events_terms *terms, struct perf_pmu_info *info, + struct parse_events_error *err); + +int perf_pmus__read_hwmon_pmus(struct list_head *pmus); + + +int evsel__hwmon_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx); +int evsel__hwmon_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread); + +#endif /* __HWMON_PMU_H */ diff --git a/tools/perf/util/include/dwarf-regs.h b/tools/perf/util/include/dwarf-regs.h index 75b28dcc8317..6f1b9f6b2466 100644 --- a/tools/perf/util/include/dwarf-regs.h +++ b/tools/perf/util/include/dwarf-regs.h @@ -2,52 +2,132 @@ #ifndef _PERF_DWARF_REGS_H_ #define _PERF_DWARF_REGS_H_ #include "annotate.h" +#include <elf.h> + +#ifndef EM_AARCH64 +#define EM_AARCH64 183 /* ARM 64 bit */ +#endif + +#ifndef EM_CSKY +#define EM_CSKY 252 /* C-SKY */ +#endif +#ifndef EF_CSKY_ABIV1 +#define EF_CSKY_ABIV1 0X10000000 +#endif +#ifndef EF_CSKY_ABIV2 +#define EF_CSKY_ABIV2 0X20000000 +#endif + +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 /* LoongArch */ +#endif + +/* EM_HOST gives the ELF machine for host, EF_HOST gives additional flags. */ +#if defined(__x86_64__) + #define EM_HOST EM_X86_64 +#elif defined(__i386__) + #define EM_HOST EM_386 +#elif defined(__aarch64__) + #define EM_HOST EM_AARCH64 +#elif defined(__arm__) + #define EM_HOST EM_ARM +#elif defined(__alpha__) + #define EM_HOST EM_ALPHA +#elif defined(__arc__) + #define EM_HOST EM_ARC +#elif defined(__AVR__) + #define EM_HOST EM_AVR +#elif defined(__AVR32__) + #define EM_HOST EM_AVR32 +#elif defined(__bfin__) + #define EM_HOST EM_BLACKFIN +#elif defined(__csky__) + #define EM_HOST EM_CSKY + #if defined(__CSKYABIV2__) + #define EF_HOST EF_CSKY_ABIV2 + #else + #define EF_HOST EF_CSKY_ABIV1 + #endif +#elif defined(__cris__) + #define EM_HOST EM_CRIS +#elif defined(__hppa__) // HP PA-RISC + #define EM_HOST EM_PARISC +#elif defined(__loongarch__) + #define EM_HOST EM_LOONGARCH +#elif defined(__mips__) + #define EM_HOST EM_MIPS +#elif defined(__m32r__) + #define EM_HOST EM_M32R +#elif defined(__microblaze__) + #define EM_HOST EM_MICROBLAZE +#elif defined(__MSP430__) + #define EM_HOST EM_MSP430 +#elif defined(__powerpc64__) + #define EM_HOST EM_PPC64 +#elif defined(__powerpc__) + #define EM_HOST EM_PPC +#elif defined(__riscv) + #define EM_HOST EM_RISCV +#elif defined(__s390x__) + #define EM_HOST EM_S390 +#elif defined(__sh__) + #define EM_HOST EM_SH +#elif defined(__sparc64__) || defined(__sparc__) + #define EM_HOST EM_SPARC +#elif defined(__xtensa__) + #define EM_HOST EM_XTENSA +#else + /* Unknown host ELF machine type. */ + #define EM_HOST EM_NONE +#endif + +#if !defined(EF_HOST) + #define EF_HOST 0 +#endif #define DWARF_REG_PC 0xd3af9c /* random number */ #define DWARF_REG_FB 0xd3affb /* random number */ -#ifdef HAVE_DWARF_SUPPORT -const char *get_arch_regstr(unsigned int n); -/* - * get_dwarf_regstr - Returns ftrace register string from DWARF regnum - * n: DWARF register number - * machine: ELF machine signature (EM_*) +#ifdef HAVE_LIBDW_SUPPORT +const char *get_csky_regstr(unsigned int n, unsigned int flags); + +/** + * get_dwarf_regstr() - Returns ftrace register string from DWARF regnum. + * @n: DWARF register number. + * @machine: ELF machine signature (EM_*). + * @flags: ELF flags for things like ABI differences. */ -const char *get_dwarf_regstr(unsigned int n, unsigned int machine); +const char *get_dwarf_regstr(unsigned int n, unsigned int machine, unsigned int flags); +int get_x86_regnum(const char *name); + +#if !defined(__x86_64__) && !defined(__i386__) int get_arch_regnum(const char *name); +#endif + /* * get_dwarf_regnum - Returns DWARF regnum from register name * name: architecture register name * machine: ELF machine signature (EM_*) */ -int get_dwarf_regnum(const char *name, unsigned int machine); +int get_dwarf_regnum(const char *name, unsigned int machine, unsigned int flags); + +void get_powerpc_regs(u32 raw_insn, int is_source, struct annotated_op_loc *op_loc); -#else /* HAVE_DWARF_SUPPORT */ +#else /* HAVE_LIBDW_SUPPORT */ static inline int get_dwarf_regnum(const char *name __maybe_unused, - unsigned int machine __maybe_unused) + unsigned int machine __maybe_unused, + unsigned int flags __maybe_unused) { return -1; } -#endif -#if !defined(__powerpc__) || !defined(HAVE_DWARF_SUPPORT) static inline void get_powerpc_regs(u32 raw_insn __maybe_unused, int is_source __maybe_unused, struct annotated_op_loc *op_loc __maybe_unused) { return; } -#else -void get_powerpc_regs(u32 raw_insn, int is_source, struct annotated_op_loc *op_loc); #endif -#ifdef HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET -/* - * Arch should support fetching the offset of a register in pt_regs - * by its name. See kernel's regs_query_register_offset in - * arch/xxx/kernel/ptrace.c. - */ -int regs_query_register_offset(const char *name); -#endif #endif diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 27d9b5c9fec8..a7c589fecb98 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -100,7 +100,7 @@ static void intel_bts_dump(struct intel_bts *bts __maybe_unused, else sz = len; printf("."); - color_fprintf(stdout, color, " %08x: ", pos); + color_fprintf(stdout, color, " %08zx: ", pos); for (i = 0; i < sz; i++) color_fprintf(stdout, color, " %02x", buf[i]); for (; i < br_sz; i++) @@ -808,7 +808,7 @@ static int intel_bts_synth_events(struct intel_bts *bts, static const char * const intel_bts_info_fmts[] = { [INTEL_BTS_PMU_TYPE] = " PMU Type %"PRId64"\n", [INTEL_BTS_TIME_SHIFT] = " Time Shift %"PRIu64"\n", - [INTEL_BTS_TIME_MULT] = " Time Muliplier %"PRIu64"\n", + [INTEL_BTS_TIME_MULT] = " Time Multiplier %"PRIu64"\n", [INTEL_BTS_TIME_ZERO] = " Time Zero %"PRIu64"\n", [INTEL_BTS_CAP_USER_TIME_ZERO] = " Cap Time Zero %"PRId64"\n", [INTEL_BTS_SNAPSHOT_MODE] = " Snapshot mode %"PRId64"\n", diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index fd2597613f3d..30be6dfe09eb 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -249,7 +249,7 @@ static void intel_pt_dump(struct intel_pt *pt __maybe_unused, else pkt_len = 1; printf("."); - color_fprintf(stdout, color, " %08x: ", pos); + color_fprintf(stdout, color, " %08zx: ", pos); for (i = 0; i < pkt_len; i++) color_fprintf(stdout, color, " %02x", buf[i]); for (; i < 16; i++) @@ -4110,7 +4110,7 @@ static int intel_pt_parse_vm_tm_corr_args(struct intel_pt *pt) static const char * const intel_pt_info_fmts[] = { [INTEL_PT_PMU_TYPE] = " PMU Type %"PRId64"\n", [INTEL_PT_TIME_SHIFT] = " Time Shift %"PRIu64"\n", - [INTEL_PT_TIME_MULT] = " Time Muliplier %"PRIu64"\n", + [INTEL_PT_TIME_MULT] = " Time Multiplier %"PRIu64"\n", [INTEL_PT_TIME_ZERO] = " Time Zero %"PRIu64"\n", [INTEL_PT_CAP_USER_TIME_ZERO] = " Cap Time Zero %"PRId64"\n", [INTEL_PT_TSC_BIT] = " TSC bit %#"PRIx64"\n", diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index fad227b625d1..4f0ac998b0cc 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -1343,7 +1343,7 @@ static int maps__set_module_path(struct maps *maps, const char *path, struct kmo * we need to update the symtab_type if needed. */ if (m->comp && is_kmod_dso(dso)) { - dso__set_symtab_type(dso, dso__symtab_type(dso)); + dso__set_symtab_type(dso, dso__symtab_type(dso)+1); dso__set_comp(dso, m->comp); } map__put(map); diff --git a/tools/perf/util/mem-events.c b/tools/perf/util/mem-events.c index 051feb93ed8d..bf5090f5220b 100644 --- a/tools/perf/util/mem-events.c +++ b/tools/perf/util/mem-events.c @@ -366,6 +366,12 @@ static const char * const mem_lvl[] = { }; static const char * const mem_lvlnum[] = { + [PERF_MEM_LVLNUM_L1] = "L1", + [PERF_MEM_LVLNUM_L2] = "L2", + [PERF_MEM_LVLNUM_L3] = "L3", + [PERF_MEM_LVLNUM_L4] = "L4", + [PERF_MEM_LVLNUM_L2_MHB] = "L2 MHB", + [PERF_MEM_LVLNUM_MSC] = "Memory-side Cache", [PERF_MEM_LVLNUM_UNC] = "Uncached", [PERF_MEM_LVLNUM_CXL] = "CXL", [PERF_MEM_LVLNUM_IO] = "I/O", @@ -448,7 +454,7 @@ int perf_mem__lvl_scnprintf(char *out, size_t sz, const struct mem_info *mem_inf if (mem_lvlnum[lvl]) l += scnprintf(out + l, sz - l, mem_lvlnum[lvl]); else - l += scnprintf(out + l, sz - l, "L%d", lvl); + l += scnprintf(out + l, sz - l, "Unknown level %d", lvl); l += scnprintf(out + l, sz - l, " %s", hit_miss); return l; diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 4dff3e925a47..46920ebadfd1 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -14,6 +14,7 @@ #include "pmus.h" #include "print-events.h" #include "smt.h" +#include "tool_pmu.h" #include "expr.h" #include "rblist.h" #include <string.h> @@ -297,8 +298,8 @@ static int setup_metric_events(const char *pmu, struct hashmap *ids, struct expr_id_data *val_ptr; /* Don't match events for the wrong hybrid PMU. */ - if (!all_pmus && ev->pmu_name && evsel__is_hybrid(ev) && - strcmp(ev->pmu_name, pmu)) + if (!all_pmus && ev->pmu && evsel__is_hybrid(ev) && + strcmp(ev->pmu->name, pmu)) continue; /* * Check for duplicate events with the same name. For @@ -673,20 +674,20 @@ static int metricgroup__build_event_string(struct strbuf *events, struct hashmap_entry *cur; size_t bkt; bool no_group = true, has_tool_events = false; - bool tool_events[PERF_TOOL_MAX] = {false}; + bool tool_events[TOOL_PMU__EVENT_MAX] = {false}; int ret = 0; #define RETURN_IF_NON_ZERO(x) do { if (x) return x; } while (0) hashmap__for_each_entry(ctx->ids, cur, bkt) { const char *sep, *rsep, *id = cur->pkey; - enum perf_tool_event ev; + enum tool_pmu_event ev; pr_debug("found event %s\n", id); /* Always move tool events outside of the group. */ - ev = perf_tool_event__from_str(id); - if (ev != PERF_TOOL_NONE) { + ev = tool_pmu__str_to_event(id); + if (ev != TOOL_PMU__EVENT_NONE) { has_tool_events = true; tool_events[ev] = true; continue; @@ -754,14 +755,14 @@ static int metricgroup__build_event_string(struct strbuf *events, if (has_tool_events) { int i; - perf_tool_event__for_each_event(i) { + tool_pmu__for_each_event(i) { if (tool_events[i]) { if (!no_group) { ret = strbuf_addch(events, ','); RETURN_IF_NON_ZERO(ret); } no_group = false; - ret = strbuf_addstr(events, perf_tool_event__to_str(i)); + ret = strbuf_addstr(events, tool_pmu__event_to_str(i)); RETURN_IF_NON_ZERO(ret); } } @@ -1147,14 +1148,14 @@ static int metric_list_cmp(void *priv __maybe_unused, const struct list_head *l, int i, left_count, right_count; left_count = hashmap__size(left->pctx->ids); - perf_tool_event__for_each_event(i) { - if (!expr__get_id(left->pctx, perf_tool_event__to_str(i), &data)) + tool_pmu__for_each_event(i) { + if (!expr__get_id(left->pctx, tool_pmu__event_to_str(i), &data)) left_count--; } right_count = hashmap__size(right->pctx->ids); - perf_tool_event__for_each_event(i) { - if (!expr__get_id(right->pctx, perf_tool_event__to_str(i), &data)) + tool_pmu__for_each_event(i) { + if (!expr__get_id(right->pctx, tool_pmu__event_to_str(i), &data)) right_count--; } @@ -1374,18 +1375,18 @@ static void metricgroup__free_metrics(struct list_head *metric_list) * to true if tool event is found. */ static void find_tool_events(const struct list_head *metric_list, - bool tool_events[PERF_TOOL_MAX]) + bool tool_events[TOOL_PMU__EVENT_MAX]) { struct metric *m; list_for_each_entry(m, metric_list, nd) { int i; - perf_tool_event__for_each_event(i) { + tool_pmu__for_each_event(i) { struct expr_id_data *data; if (!tool_events[i] && - !expr__get_id(m->pctx, perf_tool_event__to_str(i), &data)) + !expr__get_id(m->pctx, tool_pmu__event_to_str(i), &data)) tool_events[i] = true; } } @@ -1446,7 +1447,7 @@ err_out: */ static int parse_ids(bool metric_no_merge, bool fake_pmu, struct expr_parse_ctx *ids, const char *modifier, - bool group_events, const bool tool_events[PERF_TOOL_MAX], + bool group_events, const bool tool_events[TOOL_PMU__EVENT_MAX], struct evlist **out_evlist) { struct parse_events_error parse_error; @@ -1471,9 +1472,9 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, * event1 if #smt_on else 0 * Add a tool event to avoid a parse error on an empty string. */ - perf_tool_event__for_each_event(i) { + tool_pmu__for_each_event(i) { if (tool_events[i]) { - char *tmp = strdup(perf_tool_event__to_str(i)); + char *tmp = strdup(tool_pmu__event_to_str(i)); if (!tmp) return -ENOMEM; @@ -1535,7 +1536,7 @@ static int parse_groups(struct evlist *perf_evlist, struct evlist *combined_evlist = NULL; LIST_HEAD(metric_list); struct metric *m; - bool tool_events[PERF_TOOL_MAX] = {false}; + bool tool_events[TOOL_PMU__EVENT_MAX] = {false}; bool is_default = !strcmp(str, "Default"); int ret; diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 9a8be1e46d67..afeb8d815bbf 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -228,7 +228,7 @@ __add_event(struct list_head *list, int *idx, bool init_attr, const char *name, const char *metric_id, struct perf_pmu *pmu, struct list_head *config_terms, bool auto_merge_stats, - struct perf_cpu_map *cpu_list) + struct perf_cpu_map *cpu_list, u64 alternate_hw_config) { struct evsel *evsel; struct perf_cpu_map *cpus = perf_cpu_map__is_empty(cpu_list) && pmu ? pmu->cpus : cpu_list; @@ -263,7 +263,7 @@ __add_event(struct list_head *list, int *idx, evsel->core.is_pmu_core = pmu ? pmu->is_core : false; evsel->auto_merge_stats = auto_merge_stats; evsel->pmu = pmu; - evsel->pmu_name = pmu ? strdup(pmu->name) : NULL; + evsel->alternate_hw_config = alternate_hw_config; if (name) evsel->name = strdup(name); @@ -286,47 +286,19 @@ struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, { return __add_event(/*list=*/NULL, &idx, attr, /*init_attr=*/false, name, metric_id, pmu, /*config_terms=*/NULL, - /*auto_merge_stats=*/false, /*cpu_list=*/NULL); + /*auto_merge_stats=*/false, /*cpu_list=*/NULL, + /*alternate_hw_config=*/PERF_COUNT_HW_MAX); } static int add_event(struct list_head *list, int *idx, struct perf_event_attr *attr, const char *name, - const char *metric_id, struct list_head *config_terms) + const char *metric_id, struct list_head *config_terms, + u64 alternate_hw_config) { return __add_event(list, idx, attr, /*init_attr*/true, name, metric_id, /*pmu=*/NULL, config_terms, - /*auto_merge_stats=*/false, /*cpu_list=*/NULL) ? 0 : -ENOMEM; -} - -static int add_event_tool(struct list_head *list, int *idx, - enum perf_tool_event tool_event) -{ - struct evsel *evsel; - struct perf_event_attr attr = { - .type = PERF_TYPE_SOFTWARE, - .config = PERF_COUNT_SW_DUMMY, - }; - struct perf_cpu_map *cpu_list = NULL; - - if (tool_event == PERF_TOOL_DURATION_TIME) { - /* Duration time is gathered globally, pretend it is only on CPU0. */ - cpu_list = perf_cpu_map__new("0"); - } - evsel = __add_event(list, idx, &attr, /*init_attr=*/true, /*name=*/NULL, - /*metric_id=*/NULL, /*pmu=*/NULL, - /*config_terms=*/NULL, /*auto_merge_stats=*/false, - cpu_list); - perf_cpu_map__put(cpu_list); - if (!evsel) - return -ENOMEM; - evsel->tool_event = tool_event; - if (tool_event == PERF_TOOL_DURATION_TIME - || tool_event == PERF_TOOL_USER_TIME - || tool_event == PERF_TOOL_SYSTEM_TIME) { - free((char *)evsel->unit); - evsel->unit = strdup("ns"); - } - return 0; + /*auto_merge_stats=*/false, /*cpu_list=*/NULL, + alternate_hw_config) ? 0 : -ENOMEM; } /** @@ -450,7 +422,7 @@ bool parse_events__filter_pmu(const struct parse_events_state *parse_state, static int parse_events_add_pmu(struct parse_events_state *parse_state, struct list_head *list, struct perf_pmu *pmu, const struct parse_events_terms *const_parsed_terms, - bool auto_merge_stats); + bool auto_merge_stats, u64 alternate_hw_config); int parse_events_add_cache(struct list_head *list, int *idx, const char *name, struct parse_events_state *parse_state, @@ -476,7 +448,8 @@ int parse_events_add_cache(struct list_head *list, int *idx, const char *name, */ ret = parse_events_add_pmu(parse_state, list, pmu, parsed_terms, - perf_pmu__auto_merge_stats(pmu)); + perf_pmu__auto_merge_stats(pmu), + /*alternate_hw_config=*/PERF_COUNT_HW_MAX); if (ret) return ret; continue; @@ -507,7 +480,8 @@ int parse_events_add_cache(struct list_head *list, int *idx, const char *name, if (__add_event(list, idx, &attr, /*init_attr*/true, config_name ?: name, metric_id, pmu, &config_terms, /*auto_merge_stats=*/false, - /*cpu_list=*/NULL) == NULL) + /*cpu_list=*/NULL, + /*alternate_hw_config=*/PERF_COUNT_HW_MAX) == NULL) return -ENOMEM; free_config_terms(&config_terms); @@ -772,7 +746,7 @@ int parse_events_add_breakpoint(struct parse_events_state *parse_state, name = get_config_name(head_config); return add_event(list, &parse_state->idx, &attr, name, /*mertic_id=*/NULL, - &config_terms); + &config_terms, /*alternate_hw_config=*/PERF_COUNT_HW_MAX); } static int check_type_val(struct parse_events_term *term, @@ -794,7 +768,7 @@ static int check_type_val(struct parse_events_term *term, static bool config_term_shrinked; -static const char *config_term_name(enum parse_events__term_type term_type) +const char *parse_events__term_type_str(enum parse_events__term_type term_type) { /* * Update according to parse-events.l @@ -880,7 +854,7 @@ config_term_avail(enum parse_events__term_type term_type, struct parse_events_er /* term_type is validated so indexing is safe */ if (asprintf(&err_str, "'%s' is not usable in 'perf stat'", - config_term_name(term_type)) >= 0) + parse_events__term_type_str(term_type)) >= 0) parse_events_error__handle(err, -1, err_str, NULL); return false; } @@ -1004,7 +978,7 @@ do { \ case PARSE_EVENTS__TERM_TYPE_HARDWARE: default: parse_events_error__handle(err, term->err_term, - strdup(config_term_name(term->type_term)), + strdup(parse_events__term_type_str(term->type_term)), parse_events_formats_error_string(NULL)); return -EINVAL; } @@ -1072,6 +1046,7 @@ static int config_term_pmu(struct perf_event_attr *attr, if (perf_pmu__have_event(pmu, term->config)) { term->type_term = PARSE_EVENTS__TERM_TYPE_USER; term->no_value = true; + term->alternate_hw_config = true; } else { attr->type = PERF_TYPE_HARDWARE; attr->config = term->val.num; @@ -1127,8 +1102,9 @@ static int config_term_tracepoint(struct perf_event_attr *attr, default: if (err) { parse_events_error__handle(err, term->err_term, - strdup(config_term_name(term->type_term)), - strdup("valid terms: call-graph,stack-size\n")); + strdup(parse_events__term_type_str(term->type_term)), + strdup("valid terms: call-graph,stack-size\n") + ); } return -EINVAL; } @@ -1384,8 +1360,9 @@ static int __parse_events_add_numeric(struct parse_events_state *parse_state, name = get_config_name(head_config); metric_id = get_config_metric_id(head_config); ret = __add_event(list, &parse_state->idx, &attr, /*init_attr*/true, name, - metric_id, pmu, &config_terms, /*auto_merge_stats=*/false, - /*cpu_list=*/NULL) ? 0 : -ENOMEM; + metric_id, pmu, &config_terms, /*auto_merge_stats=*/false, + /*cpu_list=*/NULL, /*alternate_hw_config=*/PERF_COUNT_HW_MAX + ) == NULL ? -ENOMEM : 0; free_config_terms(&config_terms); return ret; } @@ -1421,13 +1398,6 @@ int parse_events_add_numeric(struct parse_events_state *parse_state, type, /*extended_type=*/0, config, head_config); } -int parse_events_add_tool(struct parse_events_state *parse_state, - struct list_head *list, - int tool_event) -{ - return add_event_tool(list, &parse_state->idx, tool_event); -} - static bool config_term_percore(struct list_head *config_terms) { struct evsel_config_term *term; @@ -1443,7 +1413,7 @@ static bool config_term_percore(struct list_head *config_terms) static int parse_events_add_pmu(struct parse_events_state *parse_state, struct list_head *list, struct perf_pmu *pmu, const struct parse_events_terms *const_parsed_terms, - bool auto_merge_stats) + bool auto_merge_stats, u64 alternate_hw_config) { struct perf_event_attr attr; struct perf_pmu_info info; @@ -1480,7 +1450,7 @@ static int parse_events_add_pmu(struct parse_events_state *parse_state, /*init_attr=*/true, /*name=*/NULL, /*metric_id=*/NULL, pmu, /*config_terms=*/NULL, auto_merge_stats, - /*cpu_list=*/NULL); + /*cpu_list=*/NULL, alternate_hw_config); return evsel ? 0 : -ENOMEM; } @@ -1501,7 +1471,8 @@ static int parse_events_add_pmu(struct parse_events_state *parse_state, /* Look for event names in the terms and rewrite into format based terms. */ if (perf_pmu__check_alias(pmu, &parsed_terms, - &info, &alias_rewrote_terms, err)) { + &info, &alias_rewrote_terms, + &alternate_hw_config, err)) { parse_events_terms__exit(&parsed_terms); return -EINVAL; } @@ -1537,7 +1508,9 @@ static int parse_events_add_pmu(struct parse_events_state *parse_state, return -ENOMEM; } - if (perf_pmu__config(pmu, &attr, &parsed_terms, parse_state->error)) { + /* Skip configuring hard coded terms that were applied by config_attr. */ + if (perf_pmu__config(pmu, &attr, &parsed_terms, /*apply_hardcoded=*/false, + parse_state->error)) { free_config_terms(&config_terms); parse_events_terms__exit(&parsed_terms); return -EINVAL; @@ -1546,7 +1519,8 @@ static int parse_events_add_pmu(struct parse_events_state *parse_state, evsel = __add_event(list, &parse_state->idx, &attr, /*init_attr=*/true, get_config_name(&parsed_terms), get_config_metric_id(&parsed_terms), pmu, - &config_terms, auto_merge_stats, /*cpu_list=*/NULL); + &config_terms, auto_merge_stats, /*cpu_list=*/NULL, + alternate_hw_config); if (!evsel) { parse_events_terms__exit(&parsed_terms); return -ENOMEM; @@ -1567,7 +1541,7 @@ static int parse_events_add_pmu(struct parse_events_state *parse_state, } int parse_events_multi_pmu_add(struct parse_events_state *parse_state, - const char *event_name, + const char *event_name, u64 hw_config, const struct parse_events_terms *const_parsed_terms, struct list_head **listp, void *loc_) { @@ -1620,7 +1594,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, auto_merge_stats = perf_pmu__auto_merge_stats(pmu); if (!parse_events_add_pmu(parse_state, list, pmu, - &parsed_terms, auto_merge_stats)) { + &parsed_terms, auto_merge_stats, hw_config)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); @@ -1633,7 +1607,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, if (parse_state->fake_pmu) { if (!parse_events_add_pmu(parse_state, list, perf_pmus__fake_pmu(), &parsed_terms, - /*auto_merge_stats=*/true)) { + /*auto_merge_stats=*/true, hw_config)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); @@ -1674,13 +1648,15 @@ int parse_events_multi_pmu_add_or_add_pmu(struct parse_events_state *parse_state /* Attempt to add to list assuming event_or_pmu is a PMU name. */ pmu = perf_pmus__find(event_or_pmu); if (pmu && !parse_events_add_pmu(parse_state, *listp, pmu, const_parsed_terms, - /*auto_merge_stats=*/false)) + /*auto_merge_stats=*/false, + /*alternate_hw_config=*/PERF_COUNT_HW_MAX)) return 0; if (parse_state->fake_pmu) { if (!parse_events_add_pmu(parse_state, *listp, perf_pmus__fake_pmu(), const_parsed_terms, - /*auto_merge_stats=*/false)) + /*auto_merge_stats=*/false, + /*alternate_hw_config=*/PERF_COUNT_HW_MAX)) return 0; } @@ -1693,7 +1669,8 @@ int parse_events_multi_pmu_add_or_add_pmu(struct parse_events_state *parse_state if (!parse_events_add_pmu(parse_state, *listp, pmu, const_parsed_terms, - auto_merge_stats)) { + auto_merge_stats, + /*alternate_hw_config=*/PERF_COUNT_HW_MAX)) { ok++; parse_state->wild_card_pmus = true; } @@ -1704,7 +1681,8 @@ int parse_events_multi_pmu_add_or_add_pmu(struct parse_events_state *parse_state /* Failure to add, assume event_or_pmu is an event name. */ zfree(listp); - if (!parse_events_multi_pmu_add(parse_state, event_or_pmu, const_parsed_terms, listp, loc)) + if (!parse_events_multi_pmu_add(parse_state, event_or_pmu, PERF_COUNT_HW_MAX, + const_parsed_terms, listp, loc)) return 0; if (asprintf(&help, "Unable to find PMU or event on a PMU of '%s'", event_or_pmu) < 0) @@ -1755,14 +1733,10 @@ static int parse_events__modifier_list(struct parse_events_state *parse_state, int exclude = eu | ek | eh; int exclude_GH = group ? evsel->exclude_GH : 0; - if (mod.precise) { - /* use of precise requires exclude_guest */ - eG = 1; - } if (mod.user) { if (!exclude) exclude = eu = ek = eh = 1; - if (!exclude_GH && !perf_guest) + if (!exclude_GH && !perf_guest && exclude_GH_default) eG = 1; eu = 0; } @@ -2566,7 +2540,7 @@ int parse_events_term__num(struct parse_events_term **term, struct parse_events_term temp = { .type_val = PARSE_EVENTS__TERM_TYPE_NUM, .type_term = type_term, - .config = config ? : strdup(config_term_name(type_term)), + .config = config ? : strdup(parse_events__term_type_str(type_term)), .no_value = no_value, .err_term = loc_term ? loc_term->first_column : 0, .err_val = loc_val ? loc_val->first_column : 0, @@ -2600,7 +2574,7 @@ int parse_events_term__term(struct parse_events_term **term, void *loc_term, void *loc_val) { return parse_events_term__str(term, term_lhs, NULL, - strdup(config_term_name(term_rhs)), + strdup(parse_events__term_type_str(term_rhs)), loc_term, loc_val); } @@ -2707,7 +2681,8 @@ int parse_events_terms__to_strbuf(const struct parse_events_terms *terms, struct if (ret < 0) return ret; } else if ((unsigned int)term->type_term < __PARSE_EVENTS__TERM_TYPE_NR) { - ret = strbuf_addf(sb, "%s=", config_term_name(term->type_term)); + ret = strbuf_addf(sb, "%s=", + parse_events__term_type_str(term->type_term)); if (ret < 0) return ret; } @@ -2727,7 +2702,7 @@ static void config_terms_list(char *buf, size_t buf_sz) buf[0] = '\0'; for (i = 0; i < __PARSE_EVENTS__TERM_TYPE_NR; i++) { - const char *name = config_term_name(i); + const char *name = parse_events__term_type_str(i); if (!config_term_avail(i, NULL)) continue; diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 10cc9c433116..3f4334ec6231 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -127,6 +127,12 @@ struct parse_events_term { * value is assumed to be 1. An event name also has no value. */ bool no_value; + /** + * @alternate_hw_config: config is the event name but num is an + * alternate PERF_TYPE_HARDWARE config value which is often nice for the + * sake of quick matching. + */ + bool alternate_hw_config; }; struct parse_events_error { @@ -162,6 +168,8 @@ struct parse_events_state { bool wild_card_pmus; }; +const char *parse_events__term_type_str(enum parse_events__term_type term_type); + bool parse_events__filter_pmu(const struct parse_events_state *parse_state, const struct perf_pmu *pmu); void parse_events__shrink_config_terms(void); @@ -221,9 +229,6 @@ int parse_events_add_numeric(struct parse_events_state *parse_state, u32 type, u64 config, const struct parse_events_terms *head_config, bool wildcard); -int parse_events_add_tool(struct parse_events_state *parse_state, - struct list_head *list, - int tool_event); int parse_events_add_cache(struct list_head *list, int *idx, const char *name, struct parse_events_state *parse_state, struct parse_events_terms *parsed_terms); @@ -238,7 +243,7 @@ struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, struct perf_pmu *pmu); int parse_events_multi_pmu_add(struct parse_events_state *parse_state, - const char *event_name, + const char *event_name, u64 hw_config, const struct parse_events_terms *const_parsed_terms, struct list_head **listp, void *loc); diff --git a/tools/perf/util/parse-events.l b/tools/perf/util/parse-events.l index 5a0bcd7f166a..14e5bd856a18 100644 --- a/tools/perf/util/parse-events.l +++ b/tools/perf/util/parse-events.l @@ -121,14 +121,6 @@ static int sym(yyscan_t scanner, int type, int config) return type == PERF_TYPE_HARDWARE ? PE_VALUE_SYM_HW : PE_VALUE_SYM_SW; } -static int tool(yyscan_t scanner, enum perf_tool_event event) -{ - YYSTYPE *yylval = parse_events_get_lval(scanner); - - yylval->num = event; - return PE_VALUE_SYM_TOOL; -} - static int term(yyscan_t scanner, enum parse_events__term_type type) { YYSTYPE *yylval = parse_events_get_lval(scanner); @@ -404,9 +396,6 @@ cpu-migrations|migrations { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COU alignment-faults { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_ALIGNMENT_FAULTS); } emulation-faults { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_EMULATION_FAULTS); } dummy { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_DUMMY); } -duration_time { return tool(yyscanner, PERF_TOOL_DURATION_TIME); } -user_time { return tool(yyscanner, PERF_TOOL_USER_TIME); } -system_time { return tool(yyscanner, PERF_TOOL_SYSTEM_TIME); } bpf-output { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_BPF_OUTPUT); } cgroup-switches { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CGROUP_SWITCHES); } diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index b3c51f06cbdc..f888cbb076d6 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -56,7 +56,6 @@ static void free_list_evsel(struct list_head* list_evsel) %token PE_START_EVENTS PE_START_TERMS %token PE_VALUE PE_VALUE_SYM_HW PE_VALUE_SYM_SW PE_TERM -%token PE_VALUE_SYM_TOOL %token PE_EVENT_NAME %token PE_RAW PE_NAME %token PE_MODIFIER_EVENT PE_MODIFIER_BP PE_BP_COLON PE_BP_SLASH @@ -68,7 +67,6 @@ static void free_list_evsel(struct list_head* list_evsel) %type <num> PE_VALUE %type <num> PE_VALUE_SYM_HW %type <num> PE_VALUE_SYM_SW -%type <num> PE_VALUE_SYM_TOOL %type <mod> PE_MODIFIER_EVENT %type <term_type> PE_TERM %type <num> value_sym @@ -292,7 +290,7 @@ PE_NAME sep_dc struct list_head *list; int err; - err = parse_events_multi_pmu_add(_parse_state, $1, NULL, &list, &@1); + err = parse_events_multi_pmu_add(_parse_state, $1, PERF_COUNT_HW_MAX, NULL, &list, &@1); if (err < 0) { struct parse_events_state *parse_state = _parse_state; struct parse_events_error *error = parse_state->error; @@ -350,20 +348,6 @@ value_sym sep_slash_slash_dc PE_ABORT(err); $$ = list; } -| -PE_VALUE_SYM_TOOL sep_slash_slash_dc -{ - struct list_head *list; - int err; - - list = alloc_list(); - if (!list) - YYNOMEM; - err = parse_events_add_tool(_parse_state, list, $1); - if (err) - YYNOMEM; - $$ = list; -} event_legacy_cache: PE_LEGACY_CACHE opt_event_config diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c index 5ccfe4b64cdf..0dacc133ed39 100644 --- a/tools/perf/util/pfm.c +++ b/tools/perf/util/pfm.c @@ -233,7 +233,7 @@ print_libpfm_event(const struct print_callbacks *print_cb, void *print_state, } if (is_libpfm_event_supported(name, cpus, threads)) { - print_cb->print_event(print_state, pinfo->name, topic, + print_cb->print_event(print_state, topic, pinfo->name, name, info->equiv, /*scale_unit=*/NULL, /*deprecated=*/NULL, "PFM event", @@ -267,8 +267,8 @@ print_libpfm_event(const struct print_callbacks *print_cb, void *print_state, continue; print_cb->print_event(print_state, - pinfo->name, topic, + pinfo->name, name, /*alias=*/NULL, /*scale_unit=*/NULL, /*deprecated=*/NULL, "PFM event", diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 61bdda01a05a..08a9d0bd9301 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -18,7 +18,9 @@ #include "debug.h" #include "evsel.h" #include "pmu.h" +#include "hwmon_pmu.h" #include "pmus.h" +#include "tool_pmu.h" #include <util/pmu-bison.h> #include <util/pmu-flex.h> #include "parse-events.h" @@ -817,31 +819,6 @@ static int is_sysfs_pmu_core(const char *name) return file_available(path); } -char *perf_pmu__getcpuid(struct perf_pmu *pmu) -{ - char *cpuid; - static bool printed; - - cpuid = getenv("PERF_CPUID"); - if (cpuid) - cpuid = strdup(cpuid); - if (!cpuid) - cpuid = get_cpuid_str(pmu); - if (!cpuid) - return NULL; - - if (!printed) { - pr_debug("Using CPUID %s\n", cpuid); - printed = true; - } - return cpuid; -} - -__weak const struct pmu_metrics_table *pmu_metrics_table__find(void) -{ - return perf_pmu__find_metrics_table(NULL); -} - /** * Return the length of the PMU name not including the suffix for uncore PMUs. * @@ -1168,7 +1145,7 @@ struct perf_pmu *perf_pmu__create_placeholder_core_pmu(struct list_head *core_pm return pmu; } -static bool perf_pmu__is_fake(const struct perf_pmu *pmu) +bool perf_pmu__is_fake(const struct perf_pmu *pmu) { return pmu->type == PERF_PMU_TYPE_FAKE; } @@ -1366,7 +1343,8 @@ static int pmu_config_term(const struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_term *term, struct parse_events_terms *head_terms, - bool zero, struct parse_events_error *err) + bool zero, bool apply_hardcoded, + struct parse_events_error *err) { struct perf_pmu_format *format; __u64 *vp; @@ -1380,11 +1358,46 @@ static int pmu_config_term(const struct perf_pmu *pmu, return 0; /* - * Hardcoded terms should be already in, so nothing - * to be done for them. + * Hardcoded terms are generally handled in event parsing, which + * traditionally have had to handle not having a PMU. An alias may + * have hard coded config values, optionally apply them below. */ - if (parse_events__is_hardcoded_term(term)) + if (parse_events__is_hardcoded_term(term)) { + /* Config terms set all bits in the config. */ + DECLARE_BITMAP(bits, PERF_PMU_FORMAT_BITS); + + if (!apply_hardcoded) + return 0; + + bitmap_fill(bits, PERF_PMU_FORMAT_BITS); + + switch (term->type_term) { + case PARSE_EVENTS__TERM_TYPE_CONFIG: + assert(term->type_val == PARSE_EVENTS__TERM_TYPE_NUM); + pmu_format_value(bits, term->val.num, &attr->config, zero); + break; + case PARSE_EVENTS__TERM_TYPE_CONFIG1: + assert(term->type_val == PARSE_EVENTS__TERM_TYPE_NUM); + pmu_format_value(bits, term->val.num, &attr->config1, zero); + break; + case PARSE_EVENTS__TERM_TYPE_CONFIG2: + assert(term->type_val == PARSE_EVENTS__TERM_TYPE_NUM); + pmu_format_value(bits, term->val.num, &attr->config2, zero); + break; + case PARSE_EVENTS__TERM_TYPE_CONFIG3: + assert(term->type_val == PARSE_EVENTS__TERM_TYPE_NUM); + pmu_format_value(bits, term->val.num, &attr->config3, zero); + break; + case PARSE_EVENTS__TERM_TYPE_USER: /* Not hardcoded. */ + return -EINVAL; + case PARSE_EVENTS__TERM_TYPE_NAME ... PARSE_EVENTS__TERM_TYPE_HARDWARE: + /* Skip non-config terms. */ + break; + default: + break; + } return 0; + } format = pmu_find_format(&pmu->format, term->config); if (!format) { @@ -1466,13 +1479,12 @@ static int pmu_config_term(const struct perf_pmu *pmu, if (err) { char *err_str; - parse_events_error__handle(err, term->err_val, - asprintf(&err_str, - "value too big for format (%s), maximum is %llu", - format->name, (unsigned long long)max_val) < 0 - ? strdup("value too big for format") - : err_str, - NULL); + if (asprintf(&err_str, + "value too big for format (%s), maximum is %llu", + format->name, (unsigned long long)max_val) < 0) { + err_str = strdup("value too big for format"); + } + parse_events_error__handle(err, term->err_val, err_str, /*help=*/NULL); return -EINVAL; } /* @@ -1488,12 +1500,16 @@ static int pmu_config_term(const struct perf_pmu *pmu, int perf_pmu__config_terms(const struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *terms, - bool zero, struct parse_events_error *err) + bool zero, bool apply_hardcoded, + struct parse_events_error *err) { struct parse_events_term *term; + if (perf_pmu__is_hwmon(pmu)) + return hwmon_pmu__config_terms(pmu, attr, terms, err); + list_for_each_entry(term, &terms->terms, list) { - if (pmu_config_term(pmu, attr, term, terms, zero, err)) + if (pmu_config_term(pmu, attr, term, terms, zero, apply_hardcoded, err)) return -EINVAL; } @@ -1507,6 +1523,7 @@ int perf_pmu__config_terms(const struct perf_pmu *pmu, */ int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *head_terms, + bool apply_hardcoded, struct parse_events_error *err) { bool zero = !!pmu->perf_event_attr_init_default; @@ -1515,7 +1532,7 @@ int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, if (perf_pmu__is_fake(pmu)) return 0; - return perf_pmu__config_terms(pmu, attr, head_terms, zero, err); + return perf_pmu__config_terms(pmu, attr, head_terms, zero, apply_hardcoded, err); } static struct perf_pmu_alias *pmu_find_alias(struct perf_pmu *pmu, @@ -1606,7 +1623,7 @@ static int check_info_data(struct perf_pmu *pmu, */ int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_terms, struct perf_pmu_info *info, bool *rewrote_terms, - struct parse_events_error *err) + u64 *alternate_hw_config, struct parse_events_error *err) { struct parse_events_term *term, *h; struct perf_pmu_alias *alias; @@ -1623,6 +1640,11 @@ int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_ info->scale = 0.0; info->snapshot = false; + if (perf_pmu__is_hwmon(pmu)) { + ret = hwmon_pmu__check_alias(head_terms, info, err); + goto out; + } + /* Fake PMU doesn't rewrite terms. */ if (perf_pmu__is_fake(pmu)) goto out; @@ -1638,6 +1660,7 @@ int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_ NULL); return ret; } + *rewrote_terms = true; ret = check_info_data(pmu, alias, info, err, term->err_term); if (ret) @@ -1646,6 +1669,9 @@ int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_ if (alias->per_pkg) info->per_pkg = true; + if (term->alternate_hw_config) + *alternate_hw_config = term->val.num; + list_del_init(&term->list); parse_events_term__delete(term); } @@ -1790,6 +1816,10 @@ bool perf_pmu__have_event(struct perf_pmu *pmu, const char *name) { if (!name) return false; + if (perf_pmu__is_tool(pmu) && tool_pmu__skip_event(name)) + return false; + if (perf_pmu__is_hwmon(pmu)) + return hwmon_pmu__have_event(pmu, name); if (perf_pmu__find_alias(pmu, name, /*load=*/ true) != NULL) return true; if (pmu->cpu_aliases_added || !pmu->events_table) @@ -1801,6 +1831,9 @@ size_t perf_pmu__num_events(struct perf_pmu *pmu) { size_t nr; + if (perf_pmu__is_hwmon(pmu)) + return hwmon_pmu__num_events(pmu); + pmu_aliases_parse(pmu); nr = pmu->sysfs_aliases + pmu->sys_json_aliases; @@ -1811,6 +1844,9 @@ size_t perf_pmu__num_events(struct perf_pmu *pmu) else assert(pmu->cpu_json_aliases == 0); + if (perf_pmu__is_tool(pmu)) + nr -= tool_pmu__num_skip_events(); + return pmu->selectable ? nr + 1 : nr; } @@ -1861,12 +1897,18 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, int ret = 0; struct strbuf sb; + if (perf_pmu__is_hwmon(pmu)) + return hwmon_pmu__for_each_event(pmu, state, cb); + strbuf_init(&sb, /*hint=*/ 0); pmu_aliases_parse(pmu); pmu_add_cpu_aliases(pmu); list_for_each_entry(event, &pmu->aliases, list) { size_t buf_used, pmu_name_len; + if (perf_pmu__is_tool(pmu) && tool_pmu__skip_event(event->name)) + continue; + info.pmu_name = event->pmu_name ?: pmu->name; pmu_name_len = pmu_deduped_name_len(pmu, info.pmu_name, skip_duplicate_pmus); @@ -1949,6 +1991,7 @@ bool perf_pmu__is_software(const struct perf_pmu *pmu) case PERF_TYPE_HW_CACHE: return false; case PERF_TYPE_RAW: return false; case PERF_TYPE_BREAKPOINT: return true; + case PERF_PMU_TYPE_TOOL: return true; default: break; } for (size_t i = 0; i < ARRAY_SIZE(known_sw_pmus); i++) { @@ -2200,11 +2243,6 @@ bool perf_pmu__match(const struct perf_pmu *pmu, const char *tok) (need_fnmatch && !fnmatch(tok, name, 0)); } -double __weak perf_pmu__cpu_slots_per_cycle(void) -{ - return NAN; -} - int perf_pmu__event_source_devices_scnprintf(char *pathname, size_t size) { const char *sysfs = sysfs__mountpoint(); @@ -2257,6 +2295,9 @@ int perf_pmu__pathname_fd(int dirfd, const char *pmu_name, const char *filename, void perf_pmu__delete(struct perf_pmu *pmu) { + if (perf_pmu__is_hwmon(pmu)) + hwmon_pmu__exit(pmu); + perf_pmu__del_formats(&pmu->format); perf_pmu__del_aliases(pmu); perf_pmu__del_caps(pmu); @@ -2280,7 +2321,9 @@ const char *perf_pmu__name_from_config(struct perf_pmu *pmu, u64 config) pmu_add_cpu_aliases(pmu); list_for_each_entry(event, &pmu->aliases, list) { struct perf_event_attr attr = {.config = 0,}; - int ret = perf_pmu__config(pmu, &attr, &event->terms, NULL); + + int ret = perf_pmu__config(pmu, &attr, &event->terms, /*apply_hardcoded=*/true, + /*err=*/NULL); if (ret == 0 && config == attr.config) return event->name; diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 4397c48ad569..dbed6c243a5e 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -37,6 +37,9 @@ struct perf_pmu_caps { }; enum { + PERF_PMU_TYPE_HWMON_START = 0xFFFF0000, + PERF_PMU_TYPE_HWMON_END = 0xFFFFFFFD, + PERF_PMU_TYPE_TOOL = 0xFFFFFFFE, PERF_PMU_TYPE_FAKE = 0xFFFFFFFF, }; @@ -169,6 +172,10 @@ struct perf_pmu { * exclude_host. */ bool exclude_guest; + /** + * @checked: Are the missing features checked? + */ + bool checked; } missing_features; /** @@ -206,16 +213,18 @@ typedef int (*pmu_format_callback)(void *state, const char *name, int config, void pmu_add_sys_aliases(struct perf_pmu *pmu); int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *head_terms, + bool apply_hardcoded, struct parse_events_error *error); int perf_pmu__config_terms(const struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *terms, - bool zero, struct parse_events_error *error); + bool zero, bool apply_hardcoded, + struct parse_events_error *error); __u64 perf_pmu__format_bits(struct perf_pmu *pmu, const char *name); int perf_pmu__format_type(struct perf_pmu *pmu, const char *name); int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_terms, struct perf_pmu_info *info, bool *rewrote_terms, - struct parse_events_error *err); + u64 *alternate_hw_config, struct parse_events_error *err); int perf_pmu__find_event(struct perf_pmu *pmu, const char *event, void *state, pmu_event_callback cb); void perf_pmu_format__set_value(void *format, int config, unsigned long *bits); @@ -253,8 +262,6 @@ void perf_pmu__arch_init(struct perf_pmu *pmu); void pmu_add_cpu_aliases_table(struct perf_pmu *pmu, const struct pmu_events_table *table); -char *perf_pmu__getcpuid(struct perf_pmu *pmu); -const struct pmu_metrics_table *pmu_metrics_table__find(void); bool pmu_uncore_identifier_match(const char *compat, const char *id); int perf_pmu__convert_scale(const char *scale, char **end, double *sval); @@ -268,7 +275,6 @@ void perf_pmu__warn_invalid_formats(struct perf_pmu *pmu); bool perf_pmu__match(const struct perf_pmu *pmu, const char *tok); -double perf_pmu__cpu_slots_per_cycle(void); int perf_pmu__event_source_devices_scnprintf(char *pathname, size_t size); int perf_pmu__pathname_scnprintf(char *buf, size_t size, const char *pmu_name, const char *filename); @@ -280,6 +286,8 @@ struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char struct perf_pmu *perf_pmu__create_placeholder_core_pmu(struct list_head *core_pmus); void perf_pmu__delete(struct perf_pmu *pmu); struct perf_pmu *perf_pmus__find_core_pmu(void); + const char *perf_pmu__name_from_config(struct perf_pmu *pmu, u64 config); +bool perf_pmu__is_fake(const struct perf_pmu *pmu); #endif /* __PMU_H */ diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 52109af5f2f1..b493da0d22ef 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -15,6 +15,8 @@ #include "evsel.h" #include "pmus.h" #include "pmu.h" +#include "hwmon_pmu.h" +#include "tool_pmu.h" #include "print-events.h" #include "strbuf.h" @@ -200,6 +202,7 @@ static void pmu_read_sysfs(bool core_only) int fd; DIR *dir; struct dirent *dent; + struct perf_pmu *tool_pmu; if (read_sysfs_all_pmus || (core_only && read_sysfs_core_pmus)) return; @@ -229,6 +232,11 @@ static void pmu_read_sysfs(bool core_only) pr_err("Failure to set up any core PMUs\n"); } list_sort(NULL, &core_pmus, pmus_cmp); + if (!core_only) { + tool_pmu = perf_pmus__tool_pmu(); + list_add_tail(&tool_pmu->list, &other_pmus); + perf_pmus__read_hwmon_pmus(&other_pmus); + } list_sort(NULL, &other_pmus, pmus_cmp); if (!list_empty(&core_pmus)) { read_sysfs_core_pmus = true; @@ -434,6 +442,7 @@ static int perf_pmus__print_pmu_events__callback(void *vstate, pr_err("Unexpected event %s/%s/\n", info->pmu->name, info->name); return 1; } + assert(info->pmu != NULL || info->name != NULL); s = &state->aliases[state->index]; s->pmu = info->pmu; #define COPY_STR(str) s->str = info->str ? strdup(info->str) : NULL @@ -494,8 +503,8 @@ void perf_pmus__print_pmu_events(const struct print_callbacks *print_cb, void *p goto free; print_cb->print_event(print_state, - aliases[j].pmu_name, aliases[j].topic, + aliases[j].pmu_name, aliases[j].name, aliases[j].alias, aliases[j].scale_unit, @@ -724,6 +733,13 @@ struct perf_pmu *perf_pmus__add_test_pmu(int test_sysfs_dirfd, const char *name) return perf_pmu__lookup(&other_pmus, test_sysfs_dirfd, name, /*eager_load=*/true); } +struct perf_pmu *perf_pmus__add_test_hwmon_pmu(int hwmon_dir, + const char *sysfs_name, + const char *name) +{ + return hwmon_pmu__new(&other_pmus, hwmon_dir, sysfs_name, name); +} + struct perf_pmu *perf_pmus__fake_pmu(void) { static struct perf_pmu fake = { diff --git a/tools/perf/util/pmus.h b/tools/perf/util/pmus.h index e1742b56eec7..a0cb0eb2ff97 100644 --- a/tools/perf/util/pmus.h +++ b/tools/perf/util/pmus.h @@ -30,6 +30,9 @@ bool perf_pmus__supports_extended_type(void); char *perf_pmus__default_pmu_name(void); struct perf_pmu *perf_pmus__add_test_pmu(int test_sysfs_dirfd, const char *name); +struct perf_pmu *perf_pmus__add_test_hwmon_pmu(int hwmon_dir, + const char *sysfs_name, + const char *name); struct perf_pmu *perf_pmus__fake_pmu(void); #endif /* __PMUS_H */ diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index 81e0135cddf0..a786cbfb0ff5 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -29,6 +29,7 @@ #include "tracepoint.h" #include "pfm.h" #include "thread_map.h" +#include "tool_pmu.h" #include "util.h" #define MAX_NAME_LEN 100 @@ -43,21 +44,6 @@ static const char * const event_type_descriptors[] = { "Hardware breakpoint", }; -static const struct event_symbol event_symbols_tool[PERF_TOOL_MAX] = { - [PERF_TOOL_DURATION_TIME] = { - .symbol = "duration_time", - .alias = "", - }, - [PERF_TOOL_USER_TIME] = { - .symbol = "user_time", - .alias = "", - }, - [PERF_TOOL_SYSTEM_TIME] = { - .symbol = "system_time", - .alias = "", - }, -}; - /* * Print the events from <debugfs_mount_point>/tracing/events */ @@ -341,24 +327,6 @@ int print_hwcache_events(const struct print_callbacks *print_cb, void *print_sta return 0; } -void print_tool_events(const struct print_callbacks *print_cb, void *print_state) -{ - // Start at 1 because the first enum entry means no tool event. - for (int i = 1; i < PERF_TOOL_MAX; ++i) { - print_cb->print_event(print_state, - "tool", - /*pmu_name=*/NULL, - event_symbols_tool[i].symbol, - event_symbols_tool[i].alias, - /*scale_unit=*/NULL, - /*deprecated=*/false, - "Tool event", - /*desc=*/NULL, - /*long_desc=*/NULL, - /*encoding_desc=*/NULL); - } -} - void print_symbol_events(const struct print_callbacks *print_cb, void *print_state, unsigned int type, const struct event_symbol *syms, unsigned int max) @@ -422,8 +390,6 @@ void print_events(const struct print_callbacks *print_cb, void *print_state) print_symbol_events(print_cb, print_state, PERF_TYPE_SOFTWARE, event_symbols_sw, PERF_COUNT_SW_MAX); - print_tool_events(print_cb, print_state); - print_hwcache_events(print_cb, print_state); perf_pmus__print_pmu_events(print_cb, print_state); diff --git a/tools/perf/util/print-events.h b/tools/perf/util/print-events.h index bf4290bef0cd..445efa1636c1 100644 --- a/tools/perf/util/print-events.h +++ b/tools/perf/util/print-events.h @@ -36,7 +36,6 @@ void print_sdt_events(const struct print_callbacks *print_cb, void *print_state) void print_symbol_events(const struct print_callbacks *print_cb, void *print_state, unsigned int type, const struct event_symbol *syms, unsigned int max); -void print_tool_events(const struct print_callbacks *print_cb, void *print_state); void print_tracepoint_events(const struct print_callbacks *print_cb, void *print_state); bool is_event_supported(u8 type, u64 config); diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index a17c9b8a7a79..6d51a4c98ad7 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -40,6 +40,7 @@ #include "session.h" #include "string2.h" #include "strbuf.h" +#include "parse-events.h" #include <subcmd/pager.h> #include <linux/ctype.h> @@ -51,6 +52,9 @@ #define PERFPROBE_GROUP "probe" +/* Defined in kernel/trace/trace.h */ +#define MAX_EVENT_NAME_LEN 64 + bool probe_event_dry_run; /* Dry run flag */ struct probe_conf probe_conf = { .magic_num = DEFAULT_PROBE_MAGIC_NUM }; @@ -342,7 +346,7 @@ elf_err: return mod_name; } -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT static int kernel_get_module_dso(const char *module, struct dso **pdso) { @@ -1036,6 +1040,17 @@ static int _show_one_line(FILE *fp, int l, bool skip, bool show_num) return rv; } +static int sprint_line_description(char *sbuf, size_t size, struct line_range *lr) +{ + if (!lr->function) + return snprintf(sbuf, size, "file: %s, line: %d", lr->file, lr->start); + + if (lr->file) + return snprintf(sbuf, size, "function: %s, file:%s, line: %d", lr->function, lr->file, lr->start); + + return snprintf(sbuf, size, "function: %s, line:%d", lr->function, lr->start); +} + #define show_one_line_with_num(f,l) _show_one_line(f,l,false,true) #define show_one_line(f,l) _show_one_line(f,l,false,false) #define skip_one_line(f,l) _show_one_line(f,l,true,false) @@ -1065,9 +1080,12 @@ static int __show_line_range(struct line_range *lr, const char *module, ret = debuginfo__find_line_range(dinfo, lr); if (!ret) { /* Not found, retry with an alternative */ + pr_debug2("Failed to find line range in debuginfo. Fallback to alternative\n"); ret = get_alternative_line_range(dinfo, lr, module, user); if (!ret) ret = debuginfo__find_line_range(dinfo, lr); + else /* Ignore error, we just failed to find it. */ + ret = -ENOENT; } if (dinfo->build_id) { build_id__init(&bid, dinfo->build_id, BUILD_ID_SIZE); @@ -1075,7 +1093,8 @@ static int __show_line_range(struct line_range *lr, const char *module, } debuginfo__delete(dinfo); if (ret == 0 || ret == -ENOENT) { - pr_warning("Specified source line is not found.\n"); + sprint_line_description(sbuf, sizeof(sbuf), lr); + pr_warning("Specified source line(%s) is not found.\n", sbuf); return -ENOENT; } else if (ret < 0) { pr_warning("Debuginfo analysis failed.\n"); @@ -1250,7 +1269,7 @@ out: return ret; } -#else /* !HAVE_DWARF_SUPPORT */ +#else /* !HAVE_LIBDW_SUPPORT */ static void debuginfo_cache__exit(void) { @@ -1343,30 +1362,39 @@ static bool is_c_func_name(const char *name) * * SRC[:SLN[+NUM|-ELN]] * FNC[@SRC][:SLN[+NUM|-ELN]] + * + * FNC@SRC accepts `FNC@*` which forcibly specify FNC as function name. + * SRC and FUNC can be quoted by double/single quotes. */ int parse_line_range_desc(const char *arg, struct line_range *lr) { - char *range, *file, *name = strdup(arg); + char *buf = strdup(arg); + char *p; int err; - if (!name) + if (!buf) return -ENOMEM; lr->start = 0; lr->end = INT_MAX; - range = strchr(name, ':'); - if (range) { - *range++ = '\0'; + p = strpbrk_esq(buf, ":"); + if (p) { + if (p == buf) { + semantic_error("No file/function name in '%s'.\n", p); + err = -EINVAL; + goto err; + } + *(p++) = '\0'; - err = parse_line_num(&range, &lr->start, "start line"); + err = parse_line_num(&p, &lr->start, "start line"); if (err) goto err; - if (*range == '+' || *range == '-') { - const char c = *range++; + if (*p == '+' || *p == '-') { + const char c = *(p++); - err = parse_line_num(&range, &lr->end, "end line"); + err = parse_line_num(&p, &lr->end, "end line"); if (err) goto err; @@ -1390,34 +1418,41 @@ int parse_line_range_desc(const char *arg, struct line_range *lr) " than end line.\n"); goto err; } - if (*range != '\0') { - semantic_error("Tailing with invalid str '%s'.\n", range); + if (*p != '\0') { + semantic_error("Tailing with invalid str '%s'.\n", p); goto err; } } - file = strchr(name, '@'); - if (file) { - *file = '\0'; - lr->file = strdup(++file); - if (lr->file == NULL) { - err = -ENOMEM; + p = strpbrk_esq(buf, "@"); + if (p) { + *p++ = '\0'; + if (strcmp(p, "*")) { + lr->file = strdup_esq(p); + if (lr->file == NULL) { + err = -ENOMEM; + goto err; + } + } + if (*buf != '\0') + lr->function = strdup_esq(buf); + if (!lr->function && !lr->file) { + semantic_error("Only '@*' is not allowed.\n"); + err = -EINVAL; goto err; } - lr->function = name; - } else if (strchr(name, '/') || strchr(name, '.')) - lr->file = name; - else if (is_c_func_name(name))/* We reuse it for checking funcname */ - lr->function = name; + } else if (strpbrk_esq(buf, "/.")) + lr->file = strdup_esq(buf); + else if (is_c_func_name(buf))/* We reuse it for checking funcname */ + lr->function = strdup_esq(buf); else { /* Invalid name */ - semantic_error("'%s' is not a valid function name.\n", name); + semantic_error("'%s' is not a valid function name.\n", buf); err = -EINVAL; goto err; } - return 0; err: - free(name); + free(buf); return err; } @@ -1425,19 +1460,19 @@ static int parse_perf_probe_event_name(char **arg, struct perf_probe_event *pev) { char *ptr; - ptr = strpbrk_esc(*arg, ":"); + ptr = strpbrk_esq(*arg, ":"); if (ptr) { *ptr = '\0'; if (!pev->sdt && !is_c_func_name(*arg)) goto ng_name; - pev->group = strdup_esc(*arg); + pev->group = strdup_esq(*arg); if (!pev->group) return -ENOMEM; *arg = ptr + 1; } else pev->group = NULL; - pev->event = strdup_esc(*arg); + pev->event = strdup_esq(*arg); if (pev->event == NULL) return -ENOMEM; @@ -1476,7 +1511,7 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) arg++; } - ptr = strpbrk_esc(arg, ";=@+%"); + ptr = strpbrk_esq(arg, ";=@+%"); if (pev->sdt) { if (ptr) { if (*ptr != '@') { @@ -1490,7 +1525,7 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) pev->target = build_id_cache__origname(tmp); free(tmp); } else - pev->target = strdup_esc(ptr + 1); + pev->target = strdup_esq(ptr + 1); if (!pev->target) return -ENOMEM; *ptr = '\0'; @@ -1531,7 +1566,7 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) file_spec = true; } - ptr = strpbrk_esc(arg, ";:+@%"); + ptr = strpbrk_esq(arg, ";:+@%"); if (ptr) { nc = *ptr; *ptr++ = '\0'; @@ -1540,7 +1575,7 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) if (arg[0] == '\0') tmp = NULL; else { - tmp = strdup_esc(arg); + tmp = strdup_esq(arg); if (tmp == NULL) return -ENOMEM; } @@ -1578,7 +1613,7 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) return -ENOMEM; break; } - ptr = strpbrk_esc(arg, ";:+@%"); + ptr = strpbrk_esq(arg, ";:+@%"); if (ptr) { nc = *ptr; *ptr++ = '\0'; @@ -1605,7 +1640,9 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) semantic_error("SRC@SRC is not allowed.\n"); return -EINVAL; } - pp->file = strdup_esc(arg); + if (!strcmp(arg, "*")) + break; + pp->file = strdup_esq(arg); if (pp->file == NULL) return -ENOMEM; break; @@ -2757,7 +2794,10 @@ static int get_new_event_name(char *buf, size_t len, const char *base, /* Try no suffix number */ ret = e_snprintf(buf, len, "%s%s", nbase, ret_event ? "__return" : ""); if (ret < 0) { - pr_warning("snprintf() failed: %d; the event name nbase='%s' is too long\n", ret, nbase); + pr_warning("snprintf() failed: %d; the event name '%s' is too long\n" + " Hint: Set a shorter event with syntax \"EVENT=PROBEDEF\"\n" + " EVENT: Event name (max length: %d bytes).\n", + ret, nbase, MAX_EVENT_NAME_LEN); goto out; } if (!strlist__has_entry(namelist, buf)) @@ -2777,7 +2817,10 @@ static int get_new_event_name(char *buf, size_t len, const char *base, for (i = 1; i < MAX_EVENT_INDEX; i++) { ret = e_snprintf(buf, len, "%s_%d", nbase, i); if (ret < 0) { - pr_debug("snprintf() failed: %d\n", ret); + pr_warning("Add suffix failed: %d; the event name '%s' is too long\n" + " Hint: Set a shorter event with syntax \"EVENT=PROBEDEF\"\n" + " EVENT: Event name (max length: %d bytes).\n", + ret, nbase, MAX_EVENT_NAME_LEN); goto out; } if (!strlist__has_entry(namelist, buf)) @@ -2841,7 +2884,7 @@ static int probe_trace_event__set_name(struct probe_trace_event *tev, bool allow_suffix) { const char *event, *group; - char buf[64]; + char buf[MAX_EVENT_NAME_LEN]; int ret; /* If probe_event or trace_event already have the name, reuse it */ @@ -2865,6 +2908,12 @@ static int probe_trace_event__set_name(struct probe_trace_event *tev, else group = PERFPROBE_GROUP; + if (strlen(group) >= MAX_EVENT_NAME_LEN) { + pr_err("Probe group string='%s' is too long (>= %d bytes)\n", + group, MAX_EVENT_NAME_LEN); + return -ENOMEM; + } + /* Get an unused new event name */ ret = get_new_event_name(buf, sizeof(buf), event, namelist, tev->point.retprobe, allow_suffix); @@ -3705,59 +3754,6 @@ void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs) } } -int add_perf_probe_events(struct perf_probe_event *pevs, int npevs) -{ - int ret; - - ret = init_probe_symbol_maps(pevs->uprobes); - if (ret < 0) - return ret; - - ret = convert_perf_probe_events(pevs, npevs); - if (ret == 0) - ret = apply_perf_probe_events(pevs, npevs); - - cleanup_perf_probe_events(pevs, npevs); - - exit_probe_symbol_maps(); - return ret; -} - -int del_perf_probe_events(struct strfilter *filter) -{ - int ret, ret2, ufd = -1, kfd = -1; - char *str = strfilter__string(filter); - - if (!str) - return -EINVAL; - - /* Get current event names */ - ret = probe_file__open_both(&kfd, &ufd, PF_FL_RW); - if (ret < 0) - goto out; - - ret = probe_file__del_events(kfd, filter); - if (ret < 0 && ret != -ENOENT) - goto error; - - ret2 = probe_file__del_events(ufd, filter); - if (ret2 < 0 && ret2 != -ENOENT) { - ret = ret2; - goto error; - } - ret = 0; - -error: - if (kfd >= 0) - close(kfd); - if (ufd >= 0) - close(ufd); -out: - free(str); - - return ret; -} - int show_available_funcs(const char *target, struct nsinfo *nsi, struct strfilter *_filter, bool user) { diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h index 7e3b6c3d1f74..61a5f4ff4e9c 100644 --- a/tools/perf/util/probe-event.h +++ b/tools/perf/util/probe-event.h @@ -159,7 +159,6 @@ void line_range__clear(struct line_range *lr); /* Initialize line range */ int line_range__init(struct line_range *lr); -int add_perf_probe_events(struct perf_probe_event *pevs, int npevs); int convert_perf_probe_events(struct perf_probe_event *pevs, int npevs); int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs); int show_probe_trace_events(struct perf_probe_event *pevs, int npevs); @@ -168,8 +167,6 @@ void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs); struct strfilter; -int del_perf_probe_events(struct strfilter *filter); - int show_perf_probe_event(const char *group, const char *event, struct perf_probe_event *pev, const char *module, bool use_stdout); diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c index 3d50de3217d5..ec8ac242fedb 100644 --- a/tools/perf/util/probe-file.c +++ b/tools/perf/util/probe-file.c @@ -366,25 +366,6 @@ int probe_file__del_strlist(int fd, struct strlist *namelist) return ret; } -int probe_file__del_events(int fd, struct strfilter *filter) -{ - struct strlist *namelist; - int ret; - - namelist = strlist__new(NULL, NULL); - if (!namelist) - return -ENOMEM; - - ret = probe_file__get_events(fd, filter, namelist); - if (ret < 0) - goto out; - - ret = probe_file__del_strlist(fd, namelist); -out: - strlist__delete(namelist); - return ret; -} - /* Caller must ensure to remove this entry from list */ static void probe_cache_entry__delete(struct probe_cache_entry *entry) { diff --git a/tools/perf/util/probe-file.h b/tools/perf/util/probe-file.h index 0dba88c0f5f0..c2bb6a5b9dcc 100644 --- a/tools/perf/util/probe-file.h +++ b/tools/perf/util/probe-file.h @@ -44,7 +44,6 @@ struct strlist *probe_file__get_namelist(int fd); struct strlist *probe_file__get_rawlist(int fd); int probe_file__add_event(int fd, struct probe_trace_event *tev); -int probe_file__del_events(int fd, struct strfilter *filter); int probe_file__get_events(int fd, struct strfilter *filter, struct strlist *plist); int probe_file__del_strlist(int fd, struct strlist *namelist); diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index 630e16c54ed5..7f2ee0cb43ca 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c @@ -56,7 +56,7 @@ static struct probe_trace_arg_ref *alloc_trace_arg_ref(long offs) */ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, Dwarf_Op *fb_ops, Dwarf_Die *sp_die, - unsigned int machine, + const struct probe_finder *pf, struct probe_trace_arg *tvar) { Dwarf_Attribute attr; @@ -166,7 +166,7 @@ static_var: if (!tvar) return ret2; - regs = get_dwarf_regstr(regn, machine); + regs = get_dwarf_regstr(regn, pf->e_machine, pf->e_flags); if (!regs) { /* This should be a bug in DWARF or this tool */ pr_warning("Mapping for the register number %u " @@ -451,7 +451,7 @@ static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf) dwarf_diename(vr_die)); ret = convert_variable_location(vr_die, pf->addr, pf->fb_ops, - &pf->sp_die, pf->machine, pf->tvar); + &pf->sp_die, pf, pf->tvar); if (ret == -ENOENT && pf->skip_empty_arg) /* This can be found in other place. skip it */ return 0; @@ -602,7 +602,6 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); if (ret <= 0 || nops == 0) { pf->fb_ops = NULL; -#ifdef HAVE_DWARF_CFI_SUPPORT } else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa && (pf->cfi_eh != NULL || pf->cfi_dbg != NULL)) { if ((dwarf_cfi_addrframe(pf->cfi_eh, pf->addr, &frame) != 0 && @@ -613,7 +612,6 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) free(frame); return -ENOENT; } -#endif /* HAVE_DWARF_CFI_SUPPORT */ } /* Call finder's callback handler */ @@ -1136,9 +1134,9 @@ static int debuginfo__find_probes(struct debuginfo *dbg, if (gelf_getehdr(elf, &ehdr) == NULL) return -EINVAL; - pf->machine = ehdr.e_machine; + pf->e_machine = ehdr.e_machine; + pf->e_flags = ehdr.e_flags; -#ifdef HAVE_DWARF_CFI_SUPPORT do { GElf_Shdr shdr; @@ -1148,7 +1146,6 @@ static int debuginfo__find_probes(struct debuginfo *dbg, pf->cfi_dbg = dwarf_getcfi(dbg->dbg); } while (0); -#endif /* HAVE_DWARF_CFI_SUPPORT */ ret = debuginfo__find_probe_location(dbg, pf); return ret; @@ -1175,7 +1172,7 @@ static int copy_variables_cb(Dwarf_Die *die_mem, void *data) (tag == DW_TAG_variable && vf->vars)) { if (convert_variable_location(die_mem, vf->pf->addr, vf->pf->fb_ops, &pf->sp_die, - pf->machine, NULL) == 0) { + pf, /*tvar=*/NULL) == 0) { vf->args[vf->nargs].var = (char *)dwarf_diename(die_mem); if (vf->args[vf->nargs].var == NULL) { vf->ret = -ENOMEM; @@ -1379,6 +1376,8 @@ int debuginfo__find_trace_events(struct debuginfo *dbg, if (ret >= 0 && tf.pf.skip_empty_arg) ret = fill_empty_trace_arg(pev, tf.tevs, tf.ntevs); + dwarf_cfi_end(tf.pf.cfi_eh); + if (ret < 0 || tf.ntevs == 0) { for (i = 0; i < tf.ntevs; i++) clear_probe_trace_event(&tf.tevs[i]); @@ -1404,7 +1403,7 @@ static int collect_variables_cb(Dwarf_Die *die_mem, void *data) tag == DW_TAG_variable) { ret = convert_variable_location(die_mem, af->pf.addr, af->pf.fb_ops, &af->pf.sp_die, - af->pf.machine, NULL); + &af->pf, /*tvar=*/NULL); if (ret == 0 || ret == -ERANGE) { int ret2; bool externs = !af->child; @@ -1583,8 +1582,21 @@ int debuginfo__find_probe_point(struct debuginfo *dbg, u64 addr, /* Find a corresponding function (name, baseline and baseaddr) */ if (die_find_realfunc(&cudie, (Dwarf_Addr)addr, &spdie)) { - /* Get function entry information */ - func = basefunc = dwarf_diename(&spdie); + /* + * Get function entry information. + * + * As described in the document DWARF Debugging Information + * Format Version 5, section 2.22 Linkage Names, "mangled names, + * are used in various ways, ... to distinguish multiple + * entities that have the same name". + * + * Firstly try to get distinct linkage name, if fail then + * rollback to get associated name in DIE. + */ + func = basefunc = die_get_linkage_name(&spdie); + if (!func) + func = basefunc = dwarf_diename(&spdie); + if (!func || die_entrypc(&spdie, &baseaddr) != 0 || dwarf_decl_line(&spdie, &baseline) != 0) { @@ -1863,7 +1875,11 @@ int find_source_path(const char *raw_path, const char *sbuild_id, const char *prefix = symbol_conf.source_prefix; if (sbuild_id && !prefix) { - if (!get_source_from_debuginfod(raw_path, sbuild_id, new_path)) + char prefixed_raw_path[PATH_MAX]; + + path__join(prefixed_raw_path, sizeof(prefixed_raw_path), comp_dir, raw_path); + + if (!get_source_from_debuginfod(prefixed_raw_path, sbuild_id, new_path)) return 0; } diff --git a/tools/perf/util/probe-finder.h b/tools/perf/util/probe-finder.h index 3add5ff516e1..be7b46ea2460 100644 --- a/tools/perf/util/probe-finder.h +++ b/tools/perf/util/probe-finder.h @@ -21,7 +21,7 @@ static inline int is_c_varname(const char *name) return isalpha(name[0]) || name[0] == '_'; } -#ifdef HAVE_DWARF_SUPPORT +#ifdef HAVE_LIBDW_SUPPORT #include "dwarf-aux.h" #include "debuginfo.h" @@ -63,14 +63,13 @@ struct probe_finder { struct intlist *lcache; /* Line cache for lazy match */ /* For variable searching */ -#if _ELFUTILS_PREREQ(0, 142) - /* Call Frame Information from .eh_frame */ + /* Call Frame Information from .eh_frame. Owned by this struct. */ Dwarf_CFI *cfi_eh; - /* Call Frame Information from .debug_frame */ + /* Call Frame Information from .debug_frame. Not owned. */ Dwarf_CFI *cfi_dbg; -#endif Dwarf_Op *fb_ops; /* Frame base attribute */ - unsigned int machine; /* Target machine arch */ + unsigned int e_machine; /* ELF target machine arch */ + unsigned int e_flags; /* ELF target machine flags */ struct perf_probe_arg *pvar; /* Current target variable */ struct probe_trace_arg *tvar; /* Current result variable */ bool skip_empty_arg; /* Skip non-exist args */ @@ -104,6 +103,6 @@ struct line_finder { int found; }; -#endif /* HAVE_DWARF_SUPPORT */ +#endif /* HAVE_LIBDW_SUPPORT */ #endif /*_PROBE_FINDER_H */ diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index ee3d43a7ba45..2096cdbaa53b 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -6,7 +6,7 @@ #include <linux/err.h> #include <perf/cpumap.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif #include <perf/mmap.h> #include "evlist.h" diff --git a/tools/perf/util/s390-cpumsf.c b/tools/perf/util/s390-cpumsf.c index 73846b73d0cf..30638653ad2d 100644 --- a/tools/perf/util/s390-cpumsf.c +++ b/tools/perf/util/s390-cpumsf.c @@ -345,7 +345,7 @@ static bool s390_cpumsf_trailer_show(const char *color, size_t pos, } color_fprintf(stdout, color, " [%#08zx] Trailer %c%c%c bsdes:%d" " dsdes:%d Overflow:%lld Time:%#llx\n" - "\t\tC:%d TOD:%#lx\n", + "\t\tC:%d TOD:%#llx\n", pos, te->f ? 'F' : ' ', te->a ? 'A' : ' ', diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c index 53383e97ec9d..335217bb532b 100644 --- a/tools/perf/util/s390-sample-raw.c +++ b/tools/perf/util/s390-sample-raw.c @@ -98,12 +98,12 @@ static void s390_cpumcfdg_dumptrail(const char *color, size_t offset, te.res2 = be32_to_cpu(tep->res2); color_fprintf(stdout, color, " [%#08zx] Trailer:%c%c%c%c%c" - " Cfvn:%d Csvn:%d Speed:%d TOD:%#llx\n", + " Cfvn:%d Csvn:%d Speed:%d TOD:%#lx\n", offset, te.clock_base ? 'T' : ' ', te.speed ? 'S' : ' ', te.mtda ? 'M' : ' ', te.caca ? 'C' : ' ', te.lcda ? 'L' : ' ', te.cfvn, te.csvn, te.cpu_speed, te.timestamp); - color_fprintf(stdout, color, "\t\t1:%lx 2:%lx 3:%lx TOD-Base:%#llx" + color_fprintf(stdout, color, "\t\t1:%lx 2:%lx 3:%lx TOD-Base:%#lx" " Type:%x\n\n", te.progusage1, te.progusage2, te.progusage3, te.tod_base, te.mach_type); @@ -205,7 +205,7 @@ static void s390_cpumcfdg_dump(struct perf_pmu *pmu, struct perf_sample *sample) char *ev_name = get_counter_name(ce.set, i, pmu); color_fprintf(stdout, color, - "\tCounter:%03d %s Value:%#018lx\n", i, + "\tCounter:%03zd %s Value:%#018"PRIx64"\n", i, ev_name ?: "<unknown>", be64_to_cpu(*p)); free(ev_name); } @@ -260,7 +260,7 @@ static void s390_pai_all_dump(struct evsel *evsel, struct perf_sample *sample) ev_name = get_counter_name(evsel->core.attr.config, pai_data.event_nr, evsel->pmu); - color_fprintf(stdout, color, "\tCounter:%03d %s Value:%#018lx\n", + color_fprintf(stdout, color, "\tCounter:%03d %s Value:%#018"PRIx64"\n", pai_data.event_nr, ev_name ?: "<unknown>", pai_data.event_val); free(ev_name); diff --git a/tools/perf/util/scripting-engines/trace-event-perl.c b/tools/perf/util/scripting-engines/trace-event-perl.c index e16257d5ab2c..85b7f188f729 100644 --- a/tools/perf/util/scripting-engines/trace-event-perl.c +++ b/tools/perf/util/scripting-engines/trace-event-perl.c @@ -27,7 +27,7 @@ #include <errno.h> #include <linux/bitmap.h> #include <linux/time64.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> #include <stdbool.h> /* perl needs the following define, right after including stdbool.h */ diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c index d7183134b669..8bdae066e839 100644 --- a/tools/perf/util/scripting-engines/trace-event-python.c +++ b/tools/perf/util/scripting-engines/trace-event-python.c @@ -31,7 +31,7 @@ #include <linux/compiler.h> #include <linux/time64.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif #include "../build-id.h" @@ -793,7 +793,8 @@ static int set_regs_in_dict(PyObject *dict, static void set_sym_in_dict(PyObject *dict, struct addr_location *al, const char *dso_field, const char *dso_bid_field, const char *dso_map_start, const char *dso_map_end, - const char *sym_field, const char *symoff_field) + const char *sym_field, const char *symoff_field, + const char *map_pgoff) { char sbuild_id[SBUILD_ID_SIZE]; @@ -809,6 +810,8 @@ static void set_sym_in_dict(PyObject *dict, struct addr_location *al, PyLong_FromUnsignedLong(map__start(al->map))); pydict_set_item_string_decref(dict, dso_map_end, PyLong_FromUnsignedLong(map__end(al->map))); + pydict_set_item_string_decref(dict, map_pgoff, + PyLong_FromUnsignedLongLong(map__pgoff(al->map))); } if (al->sym) { pydict_set_item_string_decref(dict, sym_field, @@ -895,7 +898,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, pydict_set_item_string_decref(dict, "comm", _PyUnicode_FromString(thread__comm_str(al->thread))); set_sym_in_dict(dict, al, "dso", "dso_bid", "dso_map_start", "dso_map_end", - "symbol", "symoff"); + "symbol", "symoff", "map_pgoff"); pydict_set_item_string_decref(dict, "callchain", callchain); @@ -920,7 +923,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyBool_FromLong(1)); set_sym_in_dict(dict_sample, addr_al, "addr_dso", "addr_dso_bid", "addr_dso_map_start", "addr_dso_map_end", - "addr_symbol", "addr_symoff"); + "addr_symbol", "addr_symoff", "addr_map_pgoff"); } if (sample->flags) diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index dbaf07bf6c5f..507e6cba9545 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -1171,18 +1171,24 @@ static int deliver_sample_value(struct evlist *evlist, union perf_event *event, struct perf_sample *sample, struct sample_read_value *v, - struct machine *machine) + struct machine *machine, + bool per_thread) { struct perf_sample_id *sid = evlist__id2sid(evlist, v->id); struct evsel *evsel; + u64 *storage = NULL; if (sid) { + storage = perf_sample_id__get_period_storage(sid, sample->tid, per_thread); + } + + if (storage) { sample->id = v->id; - sample->period = v->value - sid->period; - sid->period = v->value; + sample->period = v->value - *storage; + *storage = v->value; } - if (!sid || sid->evsel == NULL) { + if (!storage || sid->evsel == NULL) { ++evlist->stats.nr_unknown_id; return 0; } @@ -1203,17 +1209,19 @@ static int deliver_sample_group(struct evlist *evlist, union perf_event *event, struct perf_sample *sample, struct machine *machine, - u64 read_format) + u64 read_format, + bool per_thread) { int ret = -EINVAL; struct sample_read_value *v = sample->read.group.values; if (tool->dont_split_sample_group) - return deliver_sample_value(evlist, tool, event, sample, v, machine); + return deliver_sample_value(evlist, tool, event, sample, v, machine, + per_thread); sample_read_group__for_each(v, sample->read.group.nr, read_format) { ret = deliver_sample_value(evlist, tool, event, sample, v, - machine); + machine, per_thread); if (ret) break; } @@ -1228,6 +1236,7 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool /* We know evsel != NULL. */ u64 sample_type = evsel->core.attr.sample_type; u64 read_format = evsel->core.attr.read_format; + bool per_thread = perf_evsel__attr_has_per_thread_sample_period(&evsel->core); /* Standard sample delivery. */ if (!(sample_type & PERF_SAMPLE_READ)) @@ -1236,10 +1245,11 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool /* For PERF_SAMPLE_READ we have either single or group mode. */ if (read_format & PERF_FORMAT_GROUP) return deliver_sample_group(evlist, tool, event, sample, - machine, read_format); + machine, read_format, per_thread); else return deliver_sample_value(evlist, tool, event, sample, - &sample->read.one, machine); + &sample->read.one, machine, + per_thread); } static int machines__deliver_event(struct machines *machines, diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 013020f33ece..9dd60c7869a2 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -35,7 +35,7 @@ #include <linux/string.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif regex_t parent_regex; @@ -677,6 +677,102 @@ struct sort_entry sort_sym_ipc_null = { .se_width_idx = HISTC_SYMBOL_IPC, }; +/* --sort callchain_branch_predicted */ + +static int64_t +sort__callchain_branch_predicted_cmp(struct hist_entry *left __maybe_unused, + struct hist_entry *right __maybe_unused) +{ + return 0; +} + +static int hist_entry__callchain_branch_predicted_snprintf( + struct hist_entry *he, char *bf, size_t size, unsigned int width) +{ + u64 branch_count, predicted_count; + double percent = 0.0; + char str[32]; + + callchain_branch_counts(he->callchain, &branch_count, + &predicted_count, NULL, NULL); + + if (branch_count) + percent = predicted_count * 100.0 / branch_count; + + snprintf(str, sizeof(str), "%.1f%%", percent); + return repsep_snprintf(bf, size, "%-*.*s", width, width, str); +} + +struct sort_entry sort_callchain_branch_predicted = { + .se_header = "Predicted", + .se_cmp = sort__callchain_branch_predicted_cmp, + .se_snprintf = hist_entry__callchain_branch_predicted_snprintf, + .se_width_idx = HISTC_CALLCHAIN_BRANCH_PREDICTED, +}; + +/* --sort callchain_branch_abort */ + +static int64_t +sort__callchain_branch_abort_cmp(struct hist_entry *left __maybe_unused, + struct hist_entry *right __maybe_unused) +{ + return 0; +} + +static int hist_entry__callchain_branch_abort_snprintf(struct hist_entry *he, + char *bf, size_t size, + unsigned int width) +{ + u64 branch_count, abort_count; + char str[32]; + + callchain_branch_counts(he->callchain, &branch_count, + NULL, &abort_count, NULL); + + snprintf(str, sizeof(str), "%" PRId64, abort_count); + return repsep_snprintf(bf, size, "%-*.*s", width, width, str); +} + +struct sort_entry sort_callchain_branch_abort = { + .se_header = "Abort", + .se_cmp = sort__callchain_branch_abort_cmp, + .se_snprintf = hist_entry__callchain_branch_abort_snprintf, + .se_width_idx = HISTC_CALLCHAIN_BRANCH_ABORT, +}; + +/* --sort callchain_branch_cycles */ + +static int64_t +sort__callchain_branch_cycles_cmp(struct hist_entry *left __maybe_unused, + struct hist_entry *right __maybe_unused) +{ + return 0; +} + +static int hist_entry__callchain_branch_cycles_snprintf(struct hist_entry *he, + char *bf, size_t size, + unsigned int width) +{ + u64 branch_count, cycles_count, cycles = 0; + char str[32]; + + callchain_branch_counts(he->callchain, &branch_count, + NULL, NULL, &cycles_count); + + if (branch_count) + cycles = cycles_count / branch_count; + + snprintf(str, sizeof(str), "%" PRId64 "", cycles); + return repsep_snprintf(bf, size, "%-*.*s", width, width, str); +} + +struct sort_entry sort_callchain_branch_cycles = { + .se_header = "Cycles", + .se_cmp = sort__callchain_branch_cycles_cmp, + .se_snprintf = hist_entry__callchain_branch_cycles_snprintf, + .se_width_idx = HISTC_CALLCHAIN_BRANCH_CYCLES, +}; + /* --sort srcfile */ static char no_srcfile[1]; @@ -2456,6 +2552,15 @@ static struct sort_dimension bstack_sort_dimensions[] = { DIM(SORT_SYM_IPC, "ipc_lbr", sort_sym_ipc), DIM(SORT_ADDR_FROM, "addr_from", sort_addr_from), DIM(SORT_ADDR_TO, "addr_to", sort_addr_to), + DIM(SORT_CALLCHAIN_BRANCH_PREDICTED, + "callchain_branch_predicted", + sort_callchain_branch_predicted), + DIM(SORT_CALLCHAIN_BRANCH_ABORT, + "callchain_branch_abort", + sort_callchain_branch_abort), + DIM(SORT_CALLCHAIN_BRANCH_CYCLES, + "callchain_branch_cycles", + sort_callchain_branch_cycles) }; #undef DIM @@ -3484,7 +3589,13 @@ int sort_dimension__add(struct perf_hpp_list *list, const char *tok, if (!sd->name || strncasecmp(tok, sd->name, strlen(tok))) continue; - if (sort__mode != SORT_MODE__BRANCH) + if ((sort__mode != SORT_MODE__BRANCH) && + strncasecmp(tok, "callchain_branch_predicted", + strlen(tok)) && + strncasecmp(tok, "callchain_branch_abort", + strlen(tok)) && + strncasecmp(tok, "callchain_branch_cycles", + strlen(tok))) return -EINVAL; if (sd->entry == &sort_sym_from || sd->entry == &sort_sym_to) diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h index 9ff68c6786e7..a8572574e168 100644 --- a/tools/perf/util/sort.h +++ b/tools/perf/util/sort.h @@ -88,6 +88,9 @@ enum sort_type { SORT_SYM_IPC, SORT_ADDR_FROM, SORT_ADDR_TO, + SORT_CALLCHAIN_BRANCH_PREDICTED, + SORT_CALLCHAIN_BRANCH_ABORT, + SORT_CALLCHAIN_BRANCH_CYCLES, /* memory mode specific sort keys */ __SORT_MEMORY_MODE, diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index ea96e4ebad8c..53dcdf07f5a2 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -7,6 +7,7 @@ #include <perf/cpumap.h> #include "color.h" #include "counts.h" +#include "debug.h" #include "evlist.h" #include "evsel.h" #include "stat.h" @@ -21,6 +22,7 @@ #include "iostat.h" #include "pmu.h" #include "pmus.h" +#include "tool_pmu.h" #define CNTR_NOT_SUPPORTED "<not supported>" #define CNTR_NOT_COUNTED "<not counted>" @@ -71,6 +73,32 @@ static const char *aggr_header_std[] = { [AGGR_GLOBAL] = "" }; +const char *metric_threshold_classify__color(enum metric_threshold_classify thresh) +{ + const char * const colors[] = { + "", /* unknown */ + PERF_COLOR_RED, /* bad */ + PERF_COLOR_MAGENTA, /* nearly bad */ + PERF_COLOR_YELLOW, /* less good */ + PERF_COLOR_GREEN, /* good */ + }; + static_assert(ARRAY_SIZE(colors) - 1 == METRIC_THRESHOLD_GOOD, "missing enum value"); + return colors[thresh]; +} + +static const char *metric_threshold_classify__str(enum metric_threshold_classify thresh) +{ + const char * const strs[] = { + "unknown", + "bad", + "nearly bad", + "less good", + "good", + }; + static_assert(ARRAY_SIZE(strs) - 1 == METRIC_THRESHOLD_GOOD, "missing enum value"); + return strs[thresh]; +} + static void print_running_std(struct perf_stat_config *config, u64 run, u64 ena) { if (run != ena) @@ -403,13 +431,14 @@ static void do_new_line_std(struct perf_stat_config *config, } static void print_metric_std(struct perf_stat_config *config, - void *ctx, const char *color, const char *fmt, - const char *unit, double val) + void *ctx, enum metric_threshold_classify thresh, + const char *fmt, const char *unit, double val) { struct outstate *os = ctx; FILE *out = os->fh; int n; bool newline = os->newline; + const char *color = metric_threshold_classify__color(thresh); os->newline = false; @@ -441,7 +470,7 @@ static void new_line_csv(struct perf_stat_config *config, void *ctx) static void print_metric_csv(struct perf_stat_config *config __maybe_unused, void *ctx, - const char *color __maybe_unused, + enum metric_threshold_classify thresh __maybe_unused, const char *fmt, const char *unit, double val) { struct outstate *os = ctx; @@ -462,15 +491,20 @@ static void print_metric_csv(struct perf_stat_config *config __maybe_unused, static void print_metric_json(struct perf_stat_config *config __maybe_unused, void *ctx, - const char *color __maybe_unused, + enum metric_threshold_classify thresh, const char *fmt __maybe_unused, const char *unit, double val) { struct outstate *os = ctx; FILE *out = os->fh; - fprintf(out, "\"metric-value\" : \"%f\", ", val); - fprintf(out, "\"metric-unit\" : \"%s\"", unit); + if (unit) { + fprintf(out, "\"metric-value\" : \"%f\", \"metric-unit\" : \"%s\"", val, unit); + if (thresh != METRIC_THRESHOLD_UNKNOWN) { + fprintf(out, ", \"metric-threshold\" : \"%s\"", + metric_threshold_classify__str(thresh)); + } + } if (!config->metric_only) fprintf(out, "}"); } @@ -557,13 +591,14 @@ static const char *fixunit(char *buf, struct evsel *evsel, } static void print_metric_only(struct perf_stat_config *config, - void *ctx, const char *color, const char *fmt, - const char *unit, double val) + void *ctx, enum metric_threshold_classify thresh, + const char *fmt, const char *unit, double val) { struct outstate *os = ctx; FILE *out = os->fh; char buf[1024], str[1024]; unsigned mlen = config->metric_only_len; + const char *color = metric_threshold_classify__color(thresh); if (!valid_only_metric(unit)) return; @@ -580,7 +615,8 @@ static void print_metric_only(struct perf_stat_config *config, } static void print_metric_only_csv(struct perf_stat_config *config __maybe_unused, - void *ctx, const char *color __maybe_unused, + void *ctx, + enum metric_threshold_classify thresh __maybe_unused, const char *fmt, const char *unit, double val) { @@ -602,25 +638,29 @@ static void print_metric_only_csv(struct perf_stat_config *config __maybe_unused } static void print_metric_only_json(struct perf_stat_config *config __maybe_unused, - void *ctx, const char *color __maybe_unused, + void *ctx, + enum metric_threshold_classify thresh __maybe_unused, const char *fmt, const char *unit, double val) { struct outstate *os = ctx; FILE *out = os->fh; - char buf[64], *vals, *ends; + char buf[64], *ends; char tbuf[1024]; + const char *vals; if (!valid_only_metric(unit)) return; unit = fixunit(tbuf, os->evsel, unit); + if (!unit[0]) + return; snprintf(buf, sizeof(buf), fmt ?: "", val); - ends = vals = skip_spaces(buf); + vals = ends = skip_spaces(buf); while (isdigit(*ends) || *ends == '.') ends++; *ends = 0; - if (!unit[0] || !vals[0]) - return; + if (!vals[0]) + vals = "none"; fprintf(out, "%s\"%s\" : \"%s\"", os->first ? "" : ", ", unit, vals); os->first = false; } @@ -631,7 +671,8 @@ static void new_line_metric(struct perf_stat_config *config __maybe_unused, } static void print_metric_header(struct perf_stat_config *config, - void *ctx, const char *color __maybe_unused, + void *ctx, + enum metric_threshold_classify thresh __maybe_unused, const char *fmt __maybe_unused, const char *unit, double val __maybe_unused) { @@ -805,7 +846,7 @@ static void printout(struct perf_stat_config *config, struct outstate *os, if (run == 0 || ena == 0 || counter->counts->scaled == -1) { if (config->metric_only) { - pm(config, os, NULL, "", "", 0); + pm(config, os, METRIC_THRESHOLD_UNKNOWN, "", "", 0); return; } @@ -860,7 +901,7 @@ static void printout(struct perf_stat_config *config, struct outstate *os, perf_stat__print_shadow_stats(config, counter, uval, aggr_idx, &out, &config->metric_events); } else { - pm(config, os, /*color=*/NULL, /*format=*/NULL, /*unit=*/"", /*val=*/0); + pm(config, os, METRIC_THRESHOLD_UNKNOWN, /*format=*/NULL, /*unit=*/"", /*val=*/0); } if (!config->metric_only) { @@ -871,38 +912,66 @@ static void printout(struct perf_stat_config *config, struct outstate *os, static void uniquify_event_name(struct evsel *counter) { - char *new_name; - char *config; - int ret = 0; + const char *name, *pmu_name; + char *new_name, *config; + int ret; + + /* The evsel was already uniquified. */ + if (counter->uniquified_name) + return; + + /* Avoid checking to uniquify twice. */ + counter->uniquified_name = true; + + /* The evsel has a "name=" config term or is from libpfm. */ + if (counter->use_config_name || counter->is_libpfm_event) + return; + + /* Legacy no PMU event, don't uniquify. */ + if (!counter->pmu || + (counter->pmu->type < PERF_TYPE_MAX && counter->pmu->type != PERF_TYPE_RAW)) + return; - if (counter->uniquified_name || counter->use_config_name || - !counter->pmu_name || !strncmp(evsel__name(counter), counter->pmu_name, - strlen(counter->pmu_name))) + /* A sysfs or json event replacing a legacy event, don't uniquify. */ + if (counter->pmu->is_core && counter->alternate_hw_config != PERF_COUNT_HW_MAX) return; - config = strchr(counter->name, '/'); + name = evsel__name(counter); + pmu_name = counter->pmu->name; + /* Already prefixed by the PMU name. */ + if (!strncmp(name, pmu_name, strlen(pmu_name))) + return; + + config = strchr(name, '/'); if (config) { - if (asprintf(&new_name, - "%s%s", counter->pmu_name, config) > 0) { - free(counter->name); - counter->name = new_name; - } - } else { - if (evsel__is_hybrid(counter)) { - ret = asprintf(&new_name, "%s/%s/", - counter->pmu_name, counter->name); + int len = config - name; + + if (config[1] == '/') { + /* case: event// */ + ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 2); } else { - ret = asprintf(&new_name, "%s [%s]", - counter->name, counter->pmu_name); + /* case: event/.../ */ + ret = asprintf(&new_name, "%s/%.*s,%s", pmu_name, len, name, config + 1); } + } else { + config = strchr(name, ':'); + if (config) { + /* case: event:.. */ + int len = config - name; - if (ret) { - free(counter->name); - counter->name = new_name; + ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 1); + } else { + /* case: event */ + ret = asprintf(&new_name, "%s/%s/", pmu_name, name); } } - - counter->uniquified_name = true; + if (ret > 0) { + free(counter->name); + counter->name = new_name; + } else { + /* ENOMEM from asprintf. */ + counter->uniquified_name = false; + } } static bool hybrid_uniquify(struct evsel *evsel, struct perf_stat_config *config) @@ -940,15 +1009,29 @@ static bool should_skip_zero_counter(struct perf_stat_config *config, int idx; /* + * Skip unsupported default events when not verbose. (default events + * are all marked 'skippable'). + */ + if (verbose == 0 && counter->skippable && !counter->supported) + return true; + + /* * Skip value 0 when enabling --per-thread globally, * otherwise it will have too many 0 output. */ if (config->aggr_mode == AGGR_THREAD && config->system_wide) return true; - /* Tool events have the software PMU but are only gathered on 1. */ - if (evsel__is_tool(counter)) - return true; + /* + * Many tool events are only gathered on the first index, skip other + * zero values. + */ + if (evsel__is_tool(counter)) { + struct aggr_cpu_id own_id = + config->aggr_get_id(config, (struct perf_cpu){ .cpu = 0 }); + + return !aggr_cpu_id__equal(id, &own_id); + } /* * Skip value 0 when it's an uncore event and the given aggr id @@ -1559,6 +1642,31 @@ static void print_cgroup_counter(struct perf_stat_config *config, struct evlist print_metric_end(config, os); } +static void disable_uniquify(struct evlist *evlist) +{ + struct evsel *counter; + struct perf_pmu *last_pmu = NULL; + bool first = true; + + evlist__for_each_entry(evlist, counter) { + /* If PMUs vary then uniquify can be useful. */ + if (!first && counter->pmu != last_pmu) + return; + first = false; + if (counter->pmu) { + /* Allow uniquify for uncore PMUs. */ + if (!counter->pmu->is_core) + return; + /* Keep hybrid event names uniquified for clarity. */ + if (perf_pmus__num_core_pmus() > 1) + return; + } + } + evlist__for_each_entry_continue(evlist, counter) { + counter->uniquified_name = true; + } +} + void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *config, struct target *_target, struct timespec *ts, int argc, const char **argv) @@ -1572,6 +1680,8 @@ void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *conf .first = true, }; + disable_uniquify(evlist); + if (config->iostat_run) evlist->selected = evlist__first(evlist); diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index 99376c12dd8e..47718610d5d8 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -15,6 +15,7 @@ #include <linux/zalloc.h> #include "iostat.h" #include "util/hashmap.h" +#include "tool_pmu.h" struct stats walltime_nsecs_stats; struct rusage_stats ru_stats; @@ -76,7 +77,7 @@ void perf_stat__reset_shadow_stats(void) memset(&ru_stats, 0, sizeof(ru_stats)); } -static enum stat_type evsel__stat_type(const struct evsel *evsel) +static enum stat_type evsel__stat_type(struct evsel *evsel) { /* Fake perf_hw_cache_op_id values for use with evsel__match. */ u64 PERF_COUNT_hw_cache_l1d_miss = PERF_COUNT_HW_CACHE_L1D | @@ -136,23 +137,19 @@ static enum stat_type evsel__stat_type(const struct evsel *evsel) return STAT_NONE; } -static const char *get_ratio_color(const double ratios[3], double val) +static enum metric_threshold_classify get_ratio_thresh(const double ratios[3], double val) { - const char *color = PERF_COLOR_NORMAL; + assert(ratios[0] > ratios[1]); + assert(ratios[1] > ratios[2]); - if (val > ratios[0]) - color = PERF_COLOR_RED; - else if (val > ratios[1]) - color = PERF_COLOR_MAGENTA; - else if (val > ratios[2]) - color = PERF_COLOR_YELLOW; - - return color; + return val > ratios[1] + ? (val > ratios[0] ? METRIC_THRESHOLD_BAD : METRIC_THRESHOLD_NEARLY_BAD) + : (val > ratios[2] ? METRIC_THRESHOLD_LESS_GOOD : METRIC_THRESHOLD_GOOD); } static double find_stat(const struct evsel *evsel, int aggr_idx, enum stat_type type) { - const struct evsel *cur; + struct evsel *cur; int evsel_ctx = evsel_context(evsel); evlist__for_each_entry(evsel->evlist, cur) { @@ -195,17 +192,21 @@ static void print_ratio(struct perf_stat_config *config, const struct evsel *evsel, int aggr_idx, double numerator, struct perf_stat_output_ctx *out, enum stat_type denominator_type, - const double color_ratios[3], const char *unit) + const double thresh_ratios[3], const char *_unit) { double denominator = find_stat(evsel, aggr_idx, denominator_type); + double ratio = 0; + enum metric_threshold_classify thresh = METRIC_THRESHOLD_UNKNOWN; + const char *fmt = NULL; + const char *unit = NULL; if (numerator && denominator) { - double ratio = numerator / denominator * 100.0; - const char *color = get_ratio_color(color_ratios, ratio); - - out->print_metric(config, out->ctx, color, "%7.2f%%", unit, ratio); - } else - out->print_metric(config, out->ctx, NULL, NULL, unit, 0); + ratio = numerator / denominator * 100.0; + thresh = get_ratio_thresh(thresh_ratios, ratio); + fmt = "%7.2f%%"; + unit = _unit; + } + out->print_metric(config, out->ctx, thresh, fmt, unit, ratio); } static void print_stalled_cycles_front(struct perf_stat_config *config, @@ -213,9 +214,9 @@ static void print_stalled_cycles_front(struct perf_stat_config *config, int aggr_idx, double stalled, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {50.0, 30.0, 10.0}; + const double thresh_ratios[3] = {50.0, 30.0, 10.0}; - print_ratio(config, evsel, aggr_idx, stalled, out, STAT_CYCLES, color_ratios, + print_ratio(config, evsel, aggr_idx, stalled, out, STAT_CYCLES, thresh_ratios, "frontend cycles idle"); } @@ -224,9 +225,9 @@ static void print_stalled_cycles_back(struct perf_stat_config *config, int aggr_idx, double stalled, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {75.0, 50.0, 20.0}; + const double thresh_ratios[3] = {75.0, 50.0, 20.0}; - print_ratio(config, evsel, aggr_idx, stalled, out, STAT_CYCLES, color_ratios, + print_ratio(config, evsel, aggr_idx, stalled, out, STAT_CYCLES, thresh_ratios, "backend cycles idle"); } @@ -235,9 +236,9 @@ static void print_branch_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_BRANCHES, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_BRANCHES, thresh_ratios, "of all branches"); } @@ -246,9 +247,9 @@ static void print_l1d_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_L1_DCACHE, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_L1_DCACHE, thresh_ratios, "of all L1-dcache accesses"); } @@ -257,9 +258,9 @@ static void print_l1i_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_L1_ICACHE, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_L1_ICACHE, thresh_ratios, "of all L1-icache accesses"); } @@ -268,9 +269,9 @@ static void print_ll_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_LL_CACHE, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_LL_CACHE, thresh_ratios, "of all LL-cache accesses"); } @@ -279,9 +280,9 @@ static void print_dtlb_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_DTLB_CACHE, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_DTLB_CACHE, thresh_ratios, "of all dTLB cache accesses"); } @@ -290,9 +291,9 @@ static void print_itlb_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_ITLB_CACHE, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_ITLB_CACHE, thresh_ratios, "of all iTLB cache accesses"); } @@ -301,9 +302,9 @@ static void print_cache_miss(struct perf_stat_config *config, int aggr_idx, double misses, struct perf_stat_output_ctx *out) { - static const double color_ratios[3] = {20.0, 10.0, 5.0}; + const double thresh_ratios[3] = {20.0, 10.0, 5.0}; - print_ratio(config, evsel, aggr_idx, misses, out, STAT_CACHE_REFS, color_ratios, + print_ratio(config, evsel, aggr_idx, misses, out, STAT_CACHE_REFS, thresh_ratios, "of all cache refs"); } @@ -319,15 +320,16 @@ static void print_instructions(struct perf_stat_config *config, find_stat(evsel, aggr_idx, STAT_STALLED_CYCLES_BACK)); if (cycles) { - print_metric(config, ctxp, NULL, "%7.2f ", "insn per cycle", - instructions / cycles); - } else - print_metric(config, ctxp, NULL, NULL, "insn per cycle", 0); - + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, "%7.2f ", + "insn per cycle", instructions / cycles); + } else { + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, /*fmt=*/NULL, + "insn per cycle", 0); + } if (max_stalled && instructions) { out->new_line(config, ctxp); - print_metric(config, ctxp, NULL, "%7.2f ", "stalled cycles per insn", - max_stalled / instructions); + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, "%7.2f ", + "stalled cycles per insn", max_stalled / instructions); } } @@ -341,9 +343,12 @@ static void print_cycles(struct perf_stat_config *config, if (cycles && nsecs) { double ratio = cycles / nsecs; - out->print_metric(config, out->ctx, NULL, "%8.3f", "GHz", ratio); - } else - out->print_metric(config, out->ctx, NULL, NULL, "GHz", 0); + out->print_metric(config, out->ctx, METRIC_THRESHOLD_UNKNOWN, "%8.3f", + "GHz", ratio); + } else { + out->print_metric(config, out->ctx, METRIC_THRESHOLD_UNKNOWN, /*fmt=*/NULL, + "GHz", 0); + } } static void print_nsecs(struct perf_stat_config *config, @@ -356,10 +361,12 @@ static void print_nsecs(struct perf_stat_config *config, double wall_time = avg_stats(&walltime_nsecs_stats); if (wall_time) { - print_metric(config, ctxp, NULL, "%8.3f", "CPUs utilized", + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, "%8.3f", "CPUs utilized", nsecs / (wall_time * evsel->scale)); - } else - print_metric(config, ctxp, NULL, NULL, "CPUs utilized", 0); + } else { + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, /*fmt=*/NULL, + "CPUs utilized", 0); + } } static int prepare_metric(const struct metric_expr *mexp, @@ -381,26 +388,35 @@ static int prepare_metric(const struct metric_expr *mexp, double scale; switch (evsel__tool_event(metric_events[i])) { - case PERF_TOOL_DURATION_TIME: + case TOOL_PMU__EVENT_DURATION_TIME: stats = &walltime_nsecs_stats; scale = 1e-9; break; - case PERF_TOOL_USER_TIME: + case TOOL_PMU__EVENT_USER_TIME: stats = &ru_stats.ru_utime_usec_stat; scale = 1e-6; break; - case PERF_TOOL_SYSTEM_TIME: + case TOOL_PMU__EVENT_SYSTEM_TIME: stats = &ru_stats.ru_stime_usec_stat; scale = 1e-6; break; - case PERF_TOOL_NONE: + case TOOL_PMU__EVENT_NONE: pr_err("Invalid tool event 'none'"); abort(); - case PERF_TOOL_MAX: + case TOOL_PMU__EVENT_MAX: pr_err("Invalid tool event 'max'"); abort(); + case TOOL_PMU__EVENT_HAS_PMEM: + case TOOL_PMU__EVENT_NUM_CORES: + case TOOL_PMU__EVENT_NUM_CPUS: + case TOOL_PMU__EVENT_NUM_CPUS_ONLINE: + case TOOL_PMU__EVENT_NUM_DIES: + case TOOL_PMU__EVENT_NUM_PACKAGES: + case TOOL_PMU__EVENT_SLOTS: + case TOOL_PMU__EVENT_SMT_ON: + case TOOL_PMU__EVENT_SYSTEM_TSC_FREQ: default: - pr_err("Unknown tool event '%s'", evsel__name(metric_events[i])); + pr_err("Unexpected tool event '%s'", evsel__name(metric_events[i])); abort(); } val = avg_stats(stats) * scale; @@ -483,7 +499,7 @@ static void generic_metric(struct perf_stat_config *config, double ratio, scale, threshold; int i; void *ctxp = out->ctx; - const char *color = NULL; + enum metric_threshold_classify thresh = METRIC_THRESHOLD_UNKNOWN; pctx = expr__ctx_new(); if (!pctx) @@ -501,13 +517,13 @@ static void generic_metric(struct perf_stat_config *config, if (!metric_events[i]) { if (expr__parse(&ratio, pctx, metric_expr) == 0) { char *unit; - char metric_bf[64]; + char metric_bf[128]; if (metric_threshold && expr__parse(&threshold, pctx, metric_threshold) == 0 && !isnan(threshold)) { - color = fpclassify(threshold) == FP_ZERO - ? PERF_COLOR_GREEN : PERF_COLOR_RED; + thresh = fpclassify(threshold) == FP_ZERO + ? METRIC_THRESHOLD_GOOD : METRIC_THRESHOLD_BAD; } if (metric_unit && metric_name) { @@ -522,22 +538,22 @@ static void generic_metric(struct perf_stat_config *config, scnprintf(metric_bf, sizeof(metric_bf), "%s %s", unit, metric_name); - print_metric(config, ctxp, color, "%8.1f", + print_metric(config, ctxp, thresh, "%8.1f", metric_bf, ratio); } else { - print_metric(config, ctxp, color, "%8.2f", + print_metric(config, ctxp, thresh, "%8.2f", metric_name ? metric_name : out->force_header ? evsel->name : "", ratio); } } else { - print_metric(config, ctxp, color, /*unit=*/NULL, + print_metric(config, ctxp, thresh, /*fmt=*/NULL, out->force_header ? (metric_name ?: evsel->name) : "", 0); } } else { - print_metric(config, ctxp, color, /*unit=*/NULL, + print_metric(config, ctxp, thresh, /*fmt=*/NULL, out->force_header ? (metric_name ?: evsel->name) : "", 0); } @@ -573,7 +589,7 @@ static void perf_stat__print_metricgroup_header(struct perf_stat_config *config, { bool need_full_name = perf_pmus__num_core_pmus() > 1; static const char *last_name; - static const char *last_pmu; + static const struct perf_pmu *last_pmu; char full_name[64]; /* @@ -584,21 +600,21 @@ static void perf_stat__print_metricgroup_header(struct perf_stat_config *config, * different metric events. */ if (last_name && !strcmp(last_name, name)) { - if (!need_full_name || !strcmp(last_pmu, evsel->pmu_name)) { + if (!need_full_name || last_pmu != evsel->pmu) { out->print_metricgroup_header(config, ctxp, NULL); return; } } - if (need_full_name) - scnprintf(full_name, sizeof(full_name), "%s (%s)", name, evsel->pmu_name); + if (need_full_name && evsel->pmu) + scnprintf(full_name, sizeof(full_name), "%s (%s)", name, evsel->pmu->name); else scnprintf(full_name, sizeof(full_name), "%s", name); out->print_metricgroup_header(config, ctxp, full_name); last_name = name; - last_pmu = evsel->pmu_name; + last_pmu = evsel->pmu; } /** @@ -708,17 +724,21 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, if (unit != ' ') snprintf(unit_buf, sizeof(unit_buf), "%c/sec", unit); - print_metric(config, ctxp, NULL, "%8.3f", unit_buf, ratio); - } else + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, "%8.3f", + unit_buf, ratio); + } else { num = 0; + } } } perf_stat__print_shadow_stats_metricgroup(config, evsel, aggr_idx, &num, NULL, out, metric_events); - if (num == 0) - print_metric(config, ctxp, NULL, NULL, NULL, 0); + if (num == 0) { + print_metric(config, ctxp, METRIC_THRESHOLD_UNKNOWN, + /*fmt=*/NULL, /*unit=*/NULL, 0); + } } /** diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c index 0bd5467389e4..7c2ccdcc3fdb 100644 --- a/tools/perf/util/stat.c +++ b/tools/perf/util/stat.c @@ -553,7 +553,7 @@ static bool evsel__is_alias(struct evsel *evsel_a, struct evsel *evsel_b) if (evsel__is_clock(evsel_a) != evsel__is_clock(evsel_b)) return false; - return !!strcmp(evsel_a->pmu_name, evsel_b->pmu_name); + return evsel_a->pmu != evsel_b->pmu; } static void evsel__merge_aliases(struct evsel *evsel) diff --git a/tools/perf/util/stat.h b/tools/perf/util/stat.h index fd7a187551bd..6f8cff3cd39a 100644 --- a/tools/perf/util/stat.h +++ b/tools/perf/util/stat.h @@ -154,9 +154,21 @@ struct evlist; extern struct stats walltime_nsecs_stats; extern struct rusage_stats ru_stats; +enum metric_threshold_classify { + METRIC_THRESHOLD_UNKNOWN, + METRIC_THRESHOLD_BAD, + METRIC_THRESHOLD_NEARLY_BAD, + METRIC_THRESHOLD_LESS_GOOD, + METRIC_THRESHOLD_GOOD, +}; +const char *metric_threshold_classify__color(enum metric_threshold_classify thresh); + typedef void (*print_metric_t)(struct perf_stat_config *config, - void *ctx, const char *color, const char *unit, - const char *fmt, double val); + void *ctx, + enum metric_threshold_classify thresh, + const char *fmt, + const char *unit, + double val); typedef void (*new_line_t)(struct perf_stat_config *config, void *ctx); /* Used to print the display name of the Default metricgroup for now. */ diff --git a/tools/perf/util/string.c b/tools/perf/util/string.c index 116a642ad99d..308fc7ec88cc 100644 --- a/tools/perf/util/string.c +++ b/tools/perf/util/string.c @@ -263,6 +263,34 @@ char *strpbrk_esc(char *str, const char *stopset) return ptr; } +/* Like strpbrk_esc(), but not break if it is quoted with single/double quotes */ +char *strpbrk_esq(char *str, const char *stopset) +{ + char *_stopset = NULL; + char *ptr; + const char *squote = "'"; + const char *dquote = "\""; + + if (asprintf(&_stopset, "%s%c%c", stopset, *squote, *dquote) < 0) + return NULL; + + do { + ptr = strpbrk_esc(str, _stopset); + if (!ptr) + break; + if (*ptr == *squote) + ptr = strpbrk_esc(ptr + 1, squote); + else if (*ptr == *dquote) + ptr = strpbrk_esc(ptr + 1, dquote); + else + break; + str = ptr + 1; + } while (ptr); + + free(_stopset); + return ptr; +} + /* Like strdup, but do not copy a single backslash */ char *strdup_esc(const char *str) { @@ -293,6 +321,78 @@ char *strdup_esc(const char *str) return ret; } +/* Remove backslash right before quote and return next quote address. */ +static char *remove_consumed_esc(char *str, int len, int quote) +{ + char *ptr = str, *end = str + len; + + while (*ptr != quote && ptr < end) { + if (*ptr == '\\' && *(ptr + 1) == quote) { + memmove(ptr, ptr + 1, end - (ptr + 1)); + /* now *ptr is `quote`. */ + end--; + } + ptr++; + } + + return *ptr == quote ? ptr : NULL; +} + +/* + * Like strdup_esc, but keep quoted string as it is (and single backslash + * before quote is removed). If there is no closed quote, return NULL. + */ +char *strdup_esq(const char *str) +{ + char *d, *ret; + + /* If there is no quote, return normal strdup_esc() */ + d = strpbrk_esc((char *)str, "\"'"); + if (!d) + return strdup_esc(str); + + ret = strdup(str); + if (!ret) + return NULL; + + d = ret; + do { + d = strpbrk(d, "\\\"\'"); + if (!d) + break; + + if (*d == '"' || *d == '\'') { + /* This is non-escaped quote */ + int quote = *d; + int len = strlen(d + 1) + 1; + + /* + * Remove the start quote and remove consumed escape (backslash + * before quote) and remove the end quote. If there is no end + * quote, it is the input error. + */ + memmove(d, d + 1, len); + d = remove_consumed_esc(d, len, quote); + if (!d) + goto error; + memmove(d, d + 1, strlen(d + 1) + 1); + } + if (*d == '\\') { + memmove(d, d + 1, strlen(d + 1) + 1); + if (*d == '\\') { + /* double backslash -- keep the second one. */ + d++; + } + } + } while (*d != '\0'); + + return ret; + +error: + free(ret); + return NULL; +} + unsigned int hex(char c) { if (c >= '0' && c <= '9') diff --git a/tools/perf/util/string2.h b/tools/perf/util/string2.h index 52cb8ba057c7..4c8bff47cfd3 100644 --- a/tools/perf/util/string2.h +++ b/tools/perf/util/string2.h @@ -37,6 +37,8 @@ char *asprintf__tp_filter_pids(size_t npids, pid_t *pids); char *strpbrk_esc(char *str, const char *stopset); char *strdup_esc(const char *str); +char *strpbrk_esq(char *str, const char *stopset); +char *strdup_esq(const char *str); unsigned int hex(char c); char *strreplace_chars(char needle, const char *haystack, const char *replace); diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 3bbf173ad822..0037f1163919 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -257,7 +257,7 @@ void symbols__fixup_end(struct rb_root_cached *symbols, bool is_kallsyms) * like in: * ffffffffc1937000 T hdmi_driver_init [snd_hda_codec_hdmi] */ - if (prev->end == prev->start) { + if (prev->end == prev->start && prev->type != STT_NOTYPE) { const char *prev_mod; const char *curr_mod; diff --git a/tools/perf/util/syscalltbl.c b/tools/perf/util/syscalltbl.c index 6c45ded922b6..69d8dcf5cf28 100644 --- a/tools/perf/util/syscalltbl.c +++ b/tools/perf/util/syscalltbl.c @@ -46,6 +46,10 @@ static const char *const *syscalltbl_native = syscalltbl_mips_n64; #include <asm/syscalls.c> const int syscalltbl_native_max_id = SYSCALLTBL_LOONGARCH_MAX_ID; static const char *const *syscalltbl_native = syscalltbl_loongarch; +#elif defined(__riscv) +#include <asm/syscalls.c> +const int syscalltbl_native_max_id = SYSCALLTBL_RISCV_MAX_ID; +static const char *const *syscalltbl_native = syscalltbl_riscv; #else const int syscalltbl_native_max_id = 0; static const char *const syscalltbl_native[] = { diff --git a/tools/perf/util/target.h b/tools/perf/util/target.h index d582cae8e105..2ee2cc30340f 100644 --- a/tools/perf/util/target.h +++ b/tools/perf/util/target.h @@ -17,6 +17,7 @@ struct target { bool default_per_cpu; bool per_thread; bool use_bpf; + bool inherit; int initial_delay; const char *attr_map; }; diff --git a/tools/perf/util/threads.c b/tools/perf/util/threads.c index ff2b169e0085..6ca0b178fb6c 100644 --- a/tools/perf/util/threads.c +++ b/tools/perf/util/threads.c @@ -141,7 +141,7 @@ void threads__remove_all_threads(struct threads *threads) down_write(&table->lock); __threads_table_entry__set_last_match(table, NULL); - hashmap__for_each_entry_safe((&table->shard), cur, tmp, bkt) { + hashmap__for_each_entry_safe(&table->shard, cur, tmp, bkt) { struct thread *old_value; hashmap__delete(&table->shard, cur->key, /*old_key=*/NULL, &old_value); @@ -175,7 +175,7 @@ int threads__for_each_thread(struct threads *threads, size_t bkt; down_read(&table->lock); - hashmap__for_each_entry((&table->shard), cur, bkt) { + hashmap__for_each_entry(&table->shard, cur, bkt) { int rc = fn((struct thread *)cur->pvalue, data); if (rc != 0) { diff --git a/tools/perf/util/tool_pmu.c b/tools/perf/util/tool_pmu.c new file mode 100644 index 000000000000..4fb097578479 --- /dev/null +++ b/tools/perf/util/tool_pmu.c @@ -0,0 +1,505 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "cgroup.h" +#include "counts.h" +#include "cputopo.h" +#include "evsel.h" +#include "pmu.h" +#include "print-events.h" +#include "smt.h" +#include "time-utils.h" +#include "tool_pmu.h" +#include "tsc.h" +#include <api/fs/fs.h> +#include <api/io.h> +#include <internal/threadmap.h> +#include <perf/threadmap.h> +#include <fcntl.h> +#include <strings.h> + +static const char *const tool_pmu__event_names[TOOL_PMU__EVENT_MAX] = { + NULL, + "duration_time", + "user_time", + "system_time", + "has_pmem", + "num_cores", + "num_cpus", + "num_cpus_online", + "num_dies", + "num_packages", + "slots", + "smt_on", + "system_tsc_freq", +}; + +bool tool_pmu__skip_event(const char *name __maybe_unused) +{ +#if !defined(__aarch64__) + /* The slots event should only appear on arm64. */ + if (strcasecmp(name, "slots") == 0) + return true; +#endif +#if !defined(__i386__) && !defined(__x86_64__) + /* The system_tsc_freq event should only appear on x86. */ + if (strcasecmp(name, "system_tsc_freq") == 0) + return true; +#endif + return false; +} + +int tool_pmu__num_skip_events(void) +{ + int num = 0; + +#if !defined(__aarch64__) + num++; +#endif +#if !defined(__i386__) && !defined(__x86_64__) + num++; +#endif + return num; +} + +const char *tool_pmu__event_to_str(enum tool_pmu_event ev) +{ + if (ev > TOOL_PMU__EVENT_NONE && ev < TOOL_PMU__EVENT_MAX) + return tool_pmu__event_names[ev]; + + return NULL; +} + +enum tool_pmu_event tool_pmu__str_to_event(const char *str) +{ + int i; + + if (tool_pmu__skip_event(str)) + return TOOL_PMU__EVENT_NONE; + + tool_pmu__for_each_event(i) { + if (!strcasecmp(str, tool_pmu__event_names[i])) + return i; + } + return TOOL_PMU__EVENT_NONE; +} + +bool perf_pmu__is_tool(const struct perf_pmu *pmu) +{ + return pmu && pmu->type == PERF_PMU_TYPE_TOOL; +} + +bool evsel__is_tool(const struct evsel *evsel) +{ + return perf_pmu__is_tool(evsel->pmu); +} + +enum tool_pmu_event evsel__tool_event(const struct evsel *evsel) +{ + if (!evsel__is_tool(evsel)) + return TOOL_PMU__EVENT_NONE; + + return (enum tool_pmu_event)evsel->core.attr.config; +} + +const char *evsel__tool_pmu_event_name(const struct evsel *evsel) +{ + return tool_pmu__event_to_str(evsel->core.attr.config); +} + +static bool read_until_char(struct io *io, char e) +{ + int c; + + do { + c = io__get_char(io); + if (c == -1) + return false; + } while (c != e); + return true; +} + +static int read_stat_field(int fd, struct perf_cpu cpu, int field, __u64 *val) +{ + char buf[256]; + struct io io; + int i; + + io__init(&io, fd, buf, sizeof(buf)); + + /* Skip lines to relevant CPU. */ + for (i = -1; i < cpu.cpu; i++) { + if (!read_until_char(&io, '\n')) + return -EINVAL; + } + /* Skip to "cpu". */ + if (io__get_char(&io) != 'c') return -EINVAL; + if (io__get_char(&io) != 'p') return -EINVAL; + if (io__get_char(&io) != 'u') return -EINVAL; + + /* Skip N of cpuN. */ + if (!read_until_char(&io, ' ')) + return -EINVAL; + + i = 1; + while (true) { + if (io__get_dec(&io, val) != ' ') + break; + if (field == i) + return 0; + i++; + } + return -EINVAL; +} + +static int read_pid_stat_field(int fd, int field, __u64 *val) +{ + char buf[256]; + struct io io; + int c, i; + + io__init(&io, fd, buf, sizeof(buf)); + if (io__get_dec(&io, val) != ' ') + return -EINVAL; + if (field == 1) + return 0; + + /* Skip comm. */ + if (io__get_char(&io) != '(' || !read_until_char(&io, ')')) + return -EINVAL; + if (field == 2) + return -EINVAL; /* String can't be returned. */ + + /* Skip state */ + if (io__get_char(&io) != ' ' || io__get_char(&io) == -1) + return -EINVAL; + if (field == 3) + return -EINVAL; /* String can't be returned. */ + + /* Loop over numeric fields*/ + if (io__get_char(&io) != ' ') + return -EINVAL; + + i = 4; + while (true) { + c = io__get_dec(&io, val); + if (c == -1) + return -EINVAL; + if (c == -2) { + /* Assume a -ve was read */ + c = io__get_dec(&io, val); + *val *= -1; + } + if (c != ' ') + return -EINVAL; + if (field == i) + return 0; + i++; + } + return -EINVAL; +} + +int evsel__tool_pmu_prepare_open(struct evsel *evsel, + struct perf_cpu_map *cpus, + int nthreads) +{ + if ((evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || + evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) && + !evsel->start_times) { + evsel->start_times = xyarray__new(perf_cpu_map__nr(cpus), + nthreads, + sizeof(__u64)); + if (!evsel->start_times) + return -ENOMEM; + } + return 0; +} + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) + +int evsel__tool_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + int pid = -1, idx = 0, thread = 0, nthreads, err = 0, old_errno; + + if (ev == TOOL_PMU__EVENT_NUM_CPUS) + return 0; + + if (ev == TOOL_PMU__EVENT_DURATION_TIME) { + if (evsel->core.attr.sample_period) /* no sampling */ + return -EINVAL; + evsel->start_time = rdclock(); + return 0; + } + + if (evsel->cgrp) + pid = evsel->cgrp->fd; + + nthreads = perf_thread_map__nr(threads); + for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) { + for (thread = 0; thread < nthreads; thread++) { + if (thread >= nthreads) + break; + + if (!evsel->cgrp && !evsel->core.system_wide) + pid = perf_thread_map__pid(threads, thread); + + if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { + bool system = ev == TOOL_PMU__EVENT_SYSTEM_TIME; + __u64 *start_time = NULL; + int fd; + + if (evsel->core.attr.sample_period) { + /* no sampling */ + err = -EINVAL; + goto out_close; + } + if (pid > -1) { + char buf[64]; + + snprintf(buf, sizeof(buf), "/proc/%d/stat", pid); + fd = open(buf, O_RDONLY); + evsel->pid_stat = true; + } else { + fd = open("/proc/stat", O_RDONLY); + } + FD(evsel, idx, thread) = fd; + if (fd < 0) { + err = -errno; + goto out_close; + } + start_time = xyarray__entry(evsel->start_times, idx, thread); + if (pid > -1) { + err = read_pid_stat_field(fd, system ? 15 : 14, + start_time); + } else { + struct perf_cpu cpu; + + cpu = perf_cpu_map__cpu(evsel->core.cpus, idx); + err = read_stat_field(fd, cpu, system ? 3 : 1, + start_time); + } + if (err) + goto out_close; + } + + } + } + return 0; +out_close: + if (err) + threads->err_thread = thread; + + old_errno = errno; + do { + while (--thread >= 0) { + if (FD(evsel, idx, thread) >= 0) + close(FD(evsel, idx, thread)); + FD(evsel, idx, thread) = -1; + } + thread = nthreads; + } while (--idx >= 0); + errno = old_errno; + return err; +} + +#if !defined(__i386__) && !defined(__x86_64__) +u64 arch_get_tsc_freq(void) +{ + return 0; +} +#endif + +#if !defined(__aarch64__) +u64 tool_pmu__cpu_slots_per_cycle(void) +{ + return 0; +} +#endif + +static bool has_pmem(void) +{ + static bool has_pmem, cached; + const char *sysfs = sysfs__mountpoint(); + char path[PATH_MAX]; + + if (!cached) { + snprintf(path, sizeof(path), "%s/firmware/acpi/tables/NFIT", sysfs); + has_pmem = access(path, F_OK) == 0; + cached = true; + } + return has_pmem; +} + +bool tool_pmu__read_event(enum tool_pmu_event ev, u64 *result) +{ + const struct cpu_topology *topology; + + switch (ev) { + case TOOL_PMU__EVENT_HAS_PMEM: + *result = has_pmem() ? 1 : 0; + return true; + + case TOOL_PMU__EVENT_NUM_CORES: + topology = online_topology(); + *result = topology->core_cpus_lists; + return true; + + case TOOL_PMU__EVENT_NUM_CPUS: + *result = cpu__max_present_cpu().cpu; + return true; + + case TOOL_PMU__EVENT_NUM_CPUS_ONLINE: { + struct perf_cpu_map *online = cpu_map__online(); + + if (online) { + *result = perf_cpu_map__nr(online); + return true; + } + return false; + } + case TOOL_PMU__EVENT_NUM_DIES: + topology = online_topology(); + *result = topology->die_cpus_lists; + return true; + + case TOOL_PMU__EVENT_NUM_PACKAGES: + topology = online_topology(); + *result = topology->package_cpus_lists; + return true; + + case TOOL_PMU__EVENT_SLOTS: + *result = tool_pmu__cpu_slots_per_cycle(); + return *result ? true : false; + + case TOOL_PMU__EVENT_SMT_ON: + *result = smt_on() ? 1 : 0; + return true; + + case TOOL_PMU__EVENT_SYSTEM_TSC_FREQ: + *result = arch_get_tsc_freq(); + return true; + + case TOOL_PMU__EVENT_NONE: + case TOOL_PMU__EVENT_DURATION_TIME: + case TOOL_PMU__EVENT_USER_TIME: + case TOOL_PMU__EVENT_SYSTEM_TIME: + case TOOL_PMU__EVENT_MAX: + default: + return false; + } +} + +int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) +{ + __u64 *start_time, cur_time, delta_start; + u64 val; + int fd, err = 0; + struct perf_counts_values *count, *old_count = NULL; + bool adjust = false; + enum tool_pmu_event ev = evsel__tool_event(evsel); + + count = perf_counts(evsel->counts, cpu_map_idx, thread); + + switch (ev) { + case TOOL_PMU__EVENT_HAS_PMEM: + case TOOL_PMU__EVENT_NUM_CORES: + case TOOL_PMU__EVENT_NUM_CPUS: + case TOOL_PMU__EVENT_NUM_CPUS_ONLINE: + case TOOL_PMU__EVENT_NUM_DIES: + case TOOL_PMU__EVENT_NUM_PACKAGES: + case TOOL_PMU__EVENT_SLOTS: + case TOOL_PMU__EVENT_SMT_ON: + case TOOL_PMU__EVENT_SYSTEM_TSC_FREQ: + if (evsel->prev_raw_counts) + old_count = perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); + val = 0; + if (cpu_map_idx == 0 && thread == 0) { + if (!tool_pmu__read_event(ev, &val)) { + count->lost++; + val = 0; + } + } + if (old_count) { + count->val = old_count->val + val; + count->run = old_count->run + 1; + count->ena = old_count->ena + 1; + } else { + count->val = val; + count->run++; + count->ena++; + } + return 0; + case TOOL_PMU__EVENT_DURATION_TIME: + /* + * Pretend duration_time is only on the first CPU and thread, or + * else aggregation will scale duration_time by the number of + * CPUs/threads. + */ + start_time = &evsel->start_time; + if (cpu_map_idx == 0 && thread == 0) + cur_time = rdclock(); + else + cur_time = *start_time; + break; + case TOOL_PMU__EVENT_USER_TIME: + case TOOL_PMU__EVENT_SYSTEM_TIME: { + bool system = evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME; + + start_time = xyarray__entry(evsel->start_times, cpu_map_idx, thread); + fd = FD(evsel, cpu_map_idx, thread); + lseek(fd, SEEK_SET, 0); + if (evsel->pid_stat) { + /* The event exists solely on 1 CPU. */ + if (cpu_map_idx == 0) + err = read_pid_stat_field(fd, system ? 15 : 14, &cur_time); + else + cur_time = 0; + } else { + /* The event is for all threads. */ + if (thread == 0) { + struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, + cpu_map_idx); + + err = read_stat_field(fd, cpu, system ? 3 : 1, &cur_time); + } else { + cur_time = 0; + } + } + adjust = true; + break; + } + case TOOL_PMU__EVENT_NONE: + case TOOL_PMU__EVENT_MAX: + default: + err = -EINVAL; + } + if (err) + return err; + + delta_start = cur_time - *start_time; + if (adjust) { + __u64 ticks_per_sec = sysconf(_SC_CLK_TCK); + + delta_start *= 1000000000 / ticks_per_sec; + } + count->val = delta_start; + count->ena = count->run = delta_start; + count->lost = 0; + return 0; +} + +struct perf_pmu *perf_pmus__tool_pmu(void) +{ + static struct perf_pmu tool = { + .name = "tool", + .type = PERF_PMU_TYPE_TOOL, + .aliases = LIST_HEAD_INIT(tool.aliases), + .caps = LIST_HEAD_INIT(tool.caps), + .format = LIST_HEAD_INIT(tool.format), + }; + if (!tool.events_table) + tool.events_table = find_core_events_table("common", "common"); + + return &tool; +} diff --git a/tools/perf/util/tool_pmu.h b/tools/perf/util/tool_pmu.h new file mode 100644 index 000000000000..a60184859080 --- /dev/null +++ b/tools/perf/util/tool_pmu.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __TOOL_PMU_H +#define __TOOL_PMU_H + +#include "pmu.h" + +struct evsel; +struct perf_thread_map; +struct print_callbacks; + +enum tool_pmu_event { + TOOL_PMU__EVENT_NONE = 0, + TOOL_PMU__EVENT_DURATION_TIME, + TOOL_PMU__EVENT_USER_TIME, + TOOL_PMU__EVENT_SYSTEM_TIME, + TOOL_PMU__EVENT_HAS_PMEM, + TOOL_PMU__EVENT_NUM_CORES, + TOOL_PMU__EVENT_NUM_CPUS, + TOOL_PMU__EVENT_NUM_CPUS_ONLINE, + TOOL_PMU__EVENT_NUM_DIES, + TOOL_PMU__EVENT_NUM_PACKAGES, + TOOL_PMU__EVENT_SLOTS, + TOOL_PMU__EVENT_SMT_ON, + TOOL_PMU__EVENT_SYSTEM_TSC_FREQ, + + TOOL_PMU__EVENT_MAX, +}; + +#define tool_pmu__for_each_event(ev) \ + for ((ev) = TOOL_PMU__EVENT_DURATION_TIME; (ev) < TOOL_PMU__EVENT_MAX; ev++) + +const char *tool_pmu__event_to_str(enum tool_pmu_event ev); +enum tool_pmu_event tool_pmu__str_to_event(const char *str); +bool tool_pmu__skip_event(const char *name); +int tool_pmu__num_skip_events(void); + +bool tool_pmu__read_event(enum tool_pmu_event ev, u64 *result); + +u64 tool_pmu__cpu_slots_per_cycle(void); + +bool perf_pmu__is_tool(const struct perf_pmu *pmu); + +bool evsel__is_tool(const struct evsel *evsel); +enum tool_pmu_event evsel__tool_event(const struct evsel *evsel); +const char *evsel__tool_pmu_event_name(const struct evsel *evsel); +int evsel__tool_pmu_prepare_open(struct evsel *evsel, + struct perf_cpu_map *cpus, + int nthreads); +int evsel__tool_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx); +int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread); + +struct perf_pmu *perf_pmus__tool_pmu(void); + +#endif /* __TOOL_PMU_H */ diff --git a/tools/perf/util/trace-event-parse.c b/tools/perf/util/trace-event-parse.c index f0332bd3a501..41d53e1b43e7 100644 --- a/tools/perf/util/trace-event-parse.c +++ b/tools/perf/util/trace-event-parse.c @@ -12,7 +12,7 @@ #include <linux/ctype.h> #include <linux/kernel.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> static int get_common_field(struct scripting_context *context, int *offset, int *size, const char *type) @@ -116,12 +116,6 @@ void event_format__fprintf(struct tep_event *event, trace_seq_destroy(&s); } -void event_format__print(struct tep_event *event, - int cpu, void *data, int size) -{ - return event_format__fprintf(event, cpu, data, size, stdout); -} - /* * prev_state is of size long, which is 32 bits on 32 bit architectures. * As it needs to have the same bits for both 32 bit and 64 bit architectures diff --git a/tools/perf/util/trace-event-read.c b/tools/perf/util/trace-event-read.c index 1162c49b8082..ecbbb93f0185 100644 --- a/tools/perf/util/trace-event-read.c +++ b/tools/perf/util/trace-event-read.c @@ -11,7 +11,7 @@ #include <sys/stat.h> #include <sys/wait.h> #include <sys/mman.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> diff --git a/tools/perf/util/trace-event-scripting.c b/tools/perf/util/trace-event-scripting.c index bd0000300c77..5596fcda2c10 100644 --- a/tools/perf/util/trace-event-scripting.c +++ b/tools/perf/util/trace-event-scripting.c @@ -10,7 +10,7 @@ #include <string.h> #include <errno.h> #ifdef HAVE_LIBTRACEEVENT -#include <traceevent/event-parse.h> +#include <event-parse.h> #endif #include "debug.h" diff --git a/tools/perf/util/trace-event.c b/tools/perf/util/trace-event.c index 8ad75b31e09b..6a8c66c64b70 100644 --- a/tools/perf/util/trace-event.c +++ b/tools/perf/util/trace-event.c @@ -8,7 +8,7 @@ #include <fcntl.h> #include <linux/kernel.h> #include <linux/err.h> -#include <traceevent/event-parse.h> +#include <event-parse.h> #include <api/fs/tracing_path.h> #include <api/fs/fs.h> #include "trace-event.h" diff --git a/tools/perf/util/trace-event.h b/tools/perf/util/trace-event.h index bbf8b26bc8da..79b939f947dd 100644 --- a/tools/perf/util/trace-event.h +++ b/tools/perf/util/trace-event.h @@ -42,9 +42,6 @@ struct tep_event *trace_event__tp_format_id(int id); void event_format__fprintf(struct tep_event *event, int cpu, void *data, int size, FILE *fp); -void event_format__print(struct tep_event *event, - int cpu, void *data, int size); - int parse_ftrace_file(struct tep_handle *pevent, char *buf, unsigned long size); int parse_event_file(struct tep_handle *pevent, char *buf, unsigned long size, char *sys); @@ -150,7 +147,7 @@ int common_lock_depth(struct scripting_context *context); int perf_sample__sprintf_flags(u32 flags, char *str, size_t sz); #if defined(LIBTRACEEVENT_VERSION) && LIBTRACEEVENT_VERSION >= MAKE_LIBTRACEEVENT_VERSION(1, 5, 0) -#include <traceevent/event-parse.h> +#include <event-parse.h> static inline bool tep_field_is_relative(unsigned long flags) { diff --git a/tools/perf/util/tsc.c b/tools/perf/util/tsc.c index 2e33a20e1e1b..511a517ce613 100644 --- a/tools/perf/util/tsc.c +++ b/tools/perf/util/tsc.c @@ -119,7 +119,7 @@ size_t perf_event__fprintf_time_conv(union perf_event *event, FILE *fp) size_t ret; ret = fprintf(fp, "\n... Time Shift %" PRI_lu64 "\n", tc->time_shift); - ret += fprintf(fp, "... Time Muliplier %" PRI_lu64 "\n", tc->time_mult); + ret += fprintf(fp, "... Time Multiplier %" PRI_lu64 "\n", tc->time_mult); ret += fprintf(fp, "... Time Zero %" PRI_lu64 "\n", tc->time_zero); /* diff --git a/tools/perf/util/tsc.h b/tools/perf/util/tsc.h index 88fd1c4c1cb8..57ce8449647f 100644 --- a/tools/perf/util/tsc.h +++ b/tools/perf/util/tsc.h @@ -25,7 +25,7 @@ int perf_read_tsc_conversion(const struct perf_event_mmap_page *pc, u64 perf_time_to_tsc(u64 ns, struct perf_tsc_conversion *tc); u64 tsc_to_perf_time(u64 cyc, struct perf_tsc_conversion *tc); u64 rdtsc(void); -double arch_get_tsc_freq(void); +u64 arch_get_tsc_freq(void); size_t perf_event__fprintf_time_conv(union perf_event *event, FILE *fp); diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c index 9d55a13787ce..0f031eb80b4c 100644 --- a/tools/perf/util/util.c +++ b/tools/perf/util/util.c @@ -78,17 +78,23 @@ bool sysctl__nmi_watchdog_enabled(void) bool test_attr__enabled; +bool exclude_GH_default; + bool perf_host = true; bool perf_guest = false; void event_attr_init(struct perf_event_attr *attr) { + /* to capture ABI version */ + attr->size = sizeof(*attr); + + if (!exclude_GH_default) + return; + if (!perf_host) attr->exclude_host = 1; if (!perf_guest) attr->exclude_guest = 1; - /* to capture ABI version */ - attr->size = sizeof(*attr); } int mkdir_p(char *path, mode_t mode) @@ -336,91 +342,6 @@ bool perf_event_paranoid_check(int max_level) return perf_event_paranoid() <= max_level; } -static int -fetch_ubuntu_kernel_version(unsigned int *puint) -{ - ssize_t len; - size_t line_len = 0; - char *ptr, *line = NULL; - int version, patchlevel, sublevel, err; - FILE *vsig; - - if (!puint) - return 0; - - vsig = fopen("/proc/version_signature", "r"); - if (!vsig) { - pr_debug("Open /proc/version_signature failed: %s\n", - strerror(errno)); - return -1; - } - - len = getline(&line, &line_len, vsig); - fclose(vsig); - err = -1; - if (len <= 0) { - pr_debug("Reading from /proc/version_signature failed: %s\n", - strerror(errno)); - goto errout; - } - - ptr = strrchr(line, ' '); - if (!ptr) { - pr_debug("Parsing /proc/version_signature failed: %s\n", line); - goto errout; - } - - err = sscanf(ptr + 1, "%d.%d.%d", - &version, &patchlevel, &sublevel); - if (err != 3) { - pr_debug("Unable to get kernel version from /proc/version_signature '%s'\n", - line); - goto errout; - } - - *puint = (version << 16) + (patchlevel << 8) + sublevel; - err = 0; -errout: - free(line); - return err; -} - -int -fetch_kernel_version(unsigned int *puint, char *str, - size_t str_size) -{ - struct utsname utsname; - int version, patchlevel, sublevel, err; - bool int_ver_ready = false; - - if (access("/proc/version_signature", R_OK) == 0) - if (!fetch_ubuntu_kernel_version(puint)) - int_ver_ready = true; - - if (uname(&utsname)) - return -1; - - if (str && str_size) { - strncpy(str, utsname.release, str_size); - str[str_size - 1] = '\0'; - } - - if (!puint || int_ver_ready) - return 0; - - err = sscanf(utsname.release, "%d.%d.%d", - &version, &patchlevel, &sublevel); - - if (err != 3) { - pr_debug("Unable to get kernel version from uname '%s'\n", - utsname.release); - return -1; - } - - *puint = (version << 16) + (patchlevel << 8) + sublevel; - return 0; -} - int perf_tip(char **strp, const char *dirpath) { struct strlist *tips; diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h index 9966c21aaf04..3423778e39a5 100644 --- a/tools/perf/util/util.h +++ b/tools/perf/util/util.h @@ -21,6 +21,9 @@ extern const char perf_more_info_string[]; extern const char *input_name; +/* This will control if perf_{host,guest} will set attr.exclude_{host,guest}. */ +extern bool exclude_GH_default; + extern bool perf_host; extern bool perf_guest; @@ -43,14 +46,6 @@ int sysctl__max_stack(void); bool sysctl__nmi_watchdog_enabled(void); -int fetch_kernel_version(unsigned int *puint, - char *str, size_t str_sz); -#define KVER_VERSION(x) (((x) >> 16) & 0xff) -#define KVER_PATCHLEVEL(x) (((x) >> 8) & 0xff) -#define KVER_SUBLEVEL(x) ((x) & 0xff) -#define KVER_FMT "%d.%d.%d" -#define KVER_PARAM(x) KVER_VERSION(x), KVER_PATCHLEVEL(x), KVER_SUBLEVEL(x) - int perf_tip(char **strp, const char *dirpath); #ifndef HAVE_SCHED_GETCPU_SUPPORT @@ -81,13 +76,6 @@ char *perf_exe(char *buf, int len); #endif #endif -extern bool test_attr__enabled; -void test_attr__ready(void); -void test_attr__init(void); -struct perf_event_attr; -void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, - int fd, int group_fd, unsigned long flags); - struct perf_debuginfod { const char *urls; bool set; diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 3f06b6233ae4..2401e973c359 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -72,6 +72,7 @@ TARGETS += net/packetdrill TARGETS += net/rds TARGETS += net/tcp_ao TARGETS += nsfs +TARGETS += pcie_bwctrl TARGETS += perf_events TARGETS += pidfd TARGETS += pid_namespace diff --git a/tools/testing/selftests/pcie_bwctrl/Makefile b/tools/testing/selftests/pcie_bwctrl/Makefile new file mode 100644 index 000000000000..3e84e26341d1 --- /dev/null +++ b/tools/testing/selftests/pcie_bwctrl/Makefile @@ -0,0 +1,2 @@ +TEST_PROGS = set_pcie_cooling_state.sh +include ../lib.mk diff --git a/tools/testing/selftests/pcie_bwctrl/set_pcie_cooling_state.sh b/tools/testing/selftests/pcie_bwctrl/set_pcie_cooling_state.sh new file mode 100755 index 000000000000..9df606552af3 --- /dev/null +++ b/tools/testing/selftests/pcie_bwctrl/set_pcie_cooling_state.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +SYSFS= +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 +retval=0 +skipmsg="skip all tests:" + +PCIEPORTTYPE="PCIe_Port_Link_Speed" + +prerequisite() +{ + local ports + + if [ $UID != 0 ]; then + echo $skipmsg must be run as root >&2 + exit $ksft_skip + fi + + SYSFS=`mount -t sysfs | head -1 | awk '{ print $3 }'` + + if [ ! -d "$SYSFS" ]; then + echo $skipmsg sysfs is not mounted >&2 + exit $ksft_skip + fi + + if ! ls $SYSFS/class/thermal/cooling_device* > /dev/null 2>&1; then + echo $skipmsg thermal cooling devices missing >&2 + exit $ksft_skip + fi + + ports=`grep -e "^$PCIEPORTTYPE" $SYSFS/class/thermal/cooling_device*/type | wc -l` + if [ $ports -eq 0 ]; then + echo $skipmsg pcie cooling devices missing >&2 + exit $ksft_skip + fi +} + +testport= +find_pcie_port() +{ + local patt="$1" + local pcieports + local max + local cur + local delta + local bestdelta=-1 + + pcieports=`grep -l -F -e "$patt" /sys/class/thermal/cooling_device*/type` + if [ -z "$pcieports" ]; then + return + fi + pcieports=${pcieports//\/type/} + # Find the port with the highest PCIe Link Speed + for port in $pcieports; do + max=`cat $port/max_state` + cur=`cat $port/cur_state` + delta=$((max-cur)) + if [ $delta -gt $bestdelta ]; then + testport="$port" + bestdelta=$delta + fi + done +} + +sysfspcidev= +find_sysfs_pci_dev() +{ + local typefile="$1/type" + local pcidir + + pcidir="$SYSFS/bus/pci/devices/`sed -e "s|^${PCIEPORTTYPE}_||g" $typefile`" + + if [ -r "$pcidir/current_link_speed" ]; then + sysfspcidev="$pcidir/current_link_speed" + fi +} + +usage() +{ + echo "Usage $0 [ -d dev ]" + echo -e "\t-d: PCIe port BDF string (e.g., 0000:00:04.0)" +} + +pattern="$PCIEPORTTYPE" +parse_arguments() +{ + while getopts d:h opt; do + case $opt in + h) + usage "$0" + exit 0 + ;; + d) + pattern="$PCIEPORTTYPE_$OPTARG" + ;; + *) + usage "$0" + exit 0 + ;; + esac + done +} + +parse_arguments "$@" +prerequisite +find_pcie_port "$pattern" +if [ -z "$testport" ]; then + echo $skipmsg "pcie cooling device not found from sysfs" >&2 + exit $ksft_skip +fi +find_sysfs_pci_dev "$testport" +if [ -z "$sysfspcidev" ]; then + echo $skipmsg "PCIe port device not found from sysfs" >&2 + exit $ksft_skip +fi + +./set_pcie_speed.sh "$testport" "$sysfspcidev" +retval=$? + +exit $retval diff --git a/tools/testing/selftests/pcie_bwctrl/set_pcie_speed.sh b/tools/testing/selftests/pcie_bwctrl/set_pcie_speed.sh new file mode 100755 index 000000000000..584596949312 --- /dev/null +++ b/tools/testing/selftests/pcie_bwctrl/set_pcie_speed.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +set -e + +TESTNAME=set_pcie_speed + +declare -a PCIELINKSPEED=( + "2.5 GT/s PCIe" + "5.0 GT/s PCIe" + "8.0 GT/s PCIe" + "16.0 GT/s PCIe" + "32.0 GT/s PCIe" + "64.0 GT/s PCIe" +) + +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 +retval=0 + +coolingdev="$1" +statefile="$coolingdev/cur_state" +maxfile="$coolingdev/max_state" +linkspeedfile="$2" + +oldstate=`cat $statefile` +maxstate=`cat $maxfile` + +set_state() +{ + local state=$1 + local linkspeed + local expected_linkspeed + + echo $state > $statefile + + sleep 1 + + linkspeed="`cat $linkspeedfile`" + expected_linkspeed=$((maxstate-state)) + expected_str="${PCIELINKSPEED[$expected_linkspeed]}" + if [ ! "${expected_str}" = "${linkspeed}" ]; then + echo "$TESTNAME failed: expected: ${expected_str}; got ${linkspeed}" + retval=1 + fi +} + +cleanup_skip () +{ + set_state $oldstate + exit $ksft_skip +} + +trap cleanup_skip EXIT + +echo "$TESTNAME: testing states $maxstate .. $oldstate with $coolingdev" +for i in $(seq $maxstate -1 $oldstate); do + set_state "$i" +done + +trap EXIT +if [ $retval -eq 0 ]; then + echo "$TESTNAME [PASS]" +else + echo "$TESTNAME [FAIL]" +fi +exit $retval |